From 791269a282ddf1ef15cb10cc3b3069e8ac7c151f Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Wed, 24 May 2023 12:33:28 +0200 Subject: [PATCH 1/7] Add a CI job to run client's integration tests We add Keep Client's integration tests execution to the CI process. The tests will be executed after client build and unit tests pass. --- .github/workflows/client.yml | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index ffa1c5e7c3..d83c820e62 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -263,3 +263,50 @@ jobs: with: version: "2022.1.3" install-go: false + + client-integration-test: + needs: client-build-test-publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build Docker Build Image + uses: docker/build-push-action@v3 + with: + target: build-docker + tags: go-build-env + build-args: | + VERSION=${{ env.version }} + REVISION=${{ env.revision }} + load: true # load image to local registry to use it in next steps + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + context: . + + - name: Run Go Integration Tests + run: | + docker run \ + --workdir /go/src/github.com/keep-network/keep-core \ + go-build-env \ + gotestsum -- -tags=integration ./... + + - # Temp fix - move cache instead of copying (added below step and + # modified value of `cache-to`). + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + # Without the change some jobs were failing with `no space left on device` + name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache From bdcbdc764fcc246b29b6b8081275bfc8d945130f Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Wed, 24 May 2023 13:02:05 +0200 Subject: [PATCH 2/7] Share Client Docker Build Image in CI jobs We don't want to build the same image twice, we want to share it between the jobs. This is a solution where the image is saved in the disk storage and passed to another job. This solution is based on https://github.com/docker/build-push-action/issues/225#issuecomment-727639184 --- .github/workflows/client.yml | 44 ++++++++++++------------------------ 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index d83c820e62..2eb34d18ef 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -92,8 +92,15 @@ jobs: load: true # load image to local registry to use it in next steps cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/go-build-env-image.tar context: . + - name: Upload Docker Build Image + uses: actions/upload-artifact@v3 + with: + name: go-build-env-image + path: /tmp/go-build-env-image.tar + - name: Run Go tests run: | docker run \ @@ -268,31 +275,18 @@ jobs: needs: client-build-test-publish runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Cache Docker layers - uses: actions/cache@v3 + - name: Download Docker Build Image + uses: actions/download-artifact@v2 with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + name: go-build-env-image + path: /tmp/go-build-env-image.tar - - name: Build Docker Build Image - uses: docker/build-push-action@v3 - with: - target: build-docker - tags: go-build-env - build-args: | - VERSION=${{ env.version }} - REVISION=${{ env.revision }} - load: true # load image to local registry to use it in next steps - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - context: . + - name: Load Docker Build Image + run: | + docker load --input /tmp/go-build-env-image.tar - name: Run Go Integration Tests run: | @@ -300,13 +294,3 @@ jobs: --workdir /go/src/github.com/keep-network/keep-core \ go-build-env \ gotestsum -- -tags=integration ./... - - - # Temp fix - move cache instead of copying (added below step and - # modified value of `cache-to`). - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - # Without the change some jobs were failing with `no space left on device` - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache From 18891b5702a097358275f99ad3e4c2ae5cdad3c7 Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Wed, 24 May 2023 13:30:39 +0200 Subject: [PATCH 3/7] Workaround not supported multiple outputs Since build-push-action doesn't support outputing an image to multiple outputs we need to save it in a separate step. See: https://github.com/moby/buildkit/issues/1555 Once multiple outputs are supported we can replace this step with `outputs` property configured in `Build Docker Build Image`: outputs: | type=image type=docker,dest=/tmp/go-build-env-image.tar --- .github/workflows/client.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index 2eb34d18ef..293c890beb 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -92,9 +92,19 @@ jobs: load: true # load image to local registry to use it in next steps cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new - outputs: type=docker,dest=/tmp/go-build-env-image.tar context: . + # Since build-push-action doesn't support outputing an image to multiple outputs + # we need to save it in a separate step. See: https://github.com/moby/buildkit/issues/1555 + # Once multiple outputs are supported we can replace this step with + # `outputs` property configured in `Build Docker Build Image`: + # outputs: | + # type=image + # type=docker,dest=/tmp/go-build-env-image.tar + - name: Save Docker Build Image + run: | + docker save --output /tmp/go-build-env-image.tar go-build-env + - name: Upload Docker Build Image uses: actions/upload-artifact@v3 with: From 9d50739494f1f8f8fdd34d6c4469aa9b4c4fc04b Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Wed, 24 May 2023 15:37:49 +0200 Subject: [PATCH 4/7] Fix destination path for download-artifacts It expects a path to a directory not the destination file. --- .github/workflows/client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index 293c890beb..6453fc4b9d 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -292,7 +292,7 @@ jobs: uses: actions/download-artifact@v2 with: name: go-build-env-image - path: /tmp/go-build-env-image.tar + path: /tmp - name: Load Docker Build Image run: | From 1ab7976becd31f438cca41f747c9911a390f5386 Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Wed, 24 May 2023 15:38:21 +0200 Subject: [PATCH 5/7] Update actions/download-artifact to v3 We should use v3 for consistency with other actions. --- .github/workflows/client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index 6453fc4b9d..dac10cdff4 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -289,7 +289,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Download Docker Build Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: go-build-env-image path: /tmp From 4de0500e7055d319cc05f14e8d182da1388c9809 Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Wed, 24 May 2023 20:36:42 +0200 Subject: [PATCH 6/7] Define longer timeouts for blockstream electrs --- pkg/bitcoin/electrum/electrum_integration_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/bitcoin/electrum/electrum_integration_test.go b/pkg/bitcoin/electrum/electrum_integration_test.go index fb3d05a33d..3c558a827c 100644 --- a/pkg/bitcoin/electrum/electrum_integration_test.go +++ b/pkg/bitcoin/electrum/electrum_integration_test.go @@ -44,8 +44,8 @@ var testConfigs = map[string]testConfig{ clientConfig: Config{ URL: "electrum.blockstream.info:60001", Protocol: TCP, - RequestTimeout: timeout, - RequestRetryTimeout: timeout * 2, + RequestTimeout: timeout * 3, + RequestRetryTimeout: timeout * 10, }, serverImplementation: esploraElectrs, }, @@ -53,8 +53,8 @@ var testConfigs = map[string]testConfig{ clientConfig: Config{ URL: "electrum.blockstream.info:60002", Protocol: SSL, - RequestTimeout: timeout, - RequestRetryTimeout: timeout * 3, + RequestTimeout: timeout * 3, + RequestRetryTimeout: timeout * 10, }, serverImplementation: esploraElectrs, }, From 01fbbcc419604218646dd11e2d194f75d034bf0b Mon Sep 17 00:00:00 2001 From: Jakub Nowakowski Date: Mon, 29 May 2023 15:17:02 +0200 Subject: [PATCH 7/7] Update expected error message for electrumx server The server is now is running with bitcoind deamon instead of bcoin, which made the error message change. --- pkg/bitcoin/electrum/electrum_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bitcoin/electrum/electrum_integration_test.go b/pkg/bitcoin/electrum/electrum_integration_test.go index 9975215c0f..8f57cdd4de 100644 --- a/pkg/bitcoin/electrum/electrum_integration_test.go +++ b/pkg/bitcoin/electrum/electrum_integration_test.go @@ -89,7 +89,7 @@ var testConfigs = map[string]testConfig{ RequestRetryTimeout: timeout * 2, }, errorMessages: expectedErrorMessages{ - missingTransaction: "errNo: 2, errMsg: daemon error: DaemonError({'message': 'Transaction not found.', 'code': -1})", + missingTransaction: "errNo: 2, errMsg: daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})", missingBlockHeader: "errNo: 1, errMsg: height 4,294,967,295 out of range", missingTransactionInBlock: "errNo: 1, errMsg: tx aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa not in block at height 123,456", },