From 31e22a899e6bfb9d70fd9832516d72d591cf633a Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:27:36 +1000 Subject: [PATCH 01/14] Rename :soroban-dev to :future --- .../{build-soroban-dev.yml => build-future.yml} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename .github/workflows/{build-soroban-dev.yml => build-future.yml} (92%) diff --git a/.github/workflows/build-soroban-dev.yml b/.github/workflows/build-future.yml similarity index 92% rename from .github/workflows/build-soroban-dev.yml rename to .github/workflows/build-future.yml index edb9b1e5..ff5d0fcf 100644 --- a/.github/workflows/build-soroban-dev.yml +++ b/.github/workflows/build-future.yml @@ -1,6 +1,6 @@ -name: Soroban-Dev +name: Future -# The `:soroban-dev` tag points to a build containing unreleased versions of +# The `:future` tag points to a build containing unreleased versions of # software that have been informally released to the futurenet network. on: @@ -34,7 +34,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: arch: amd64 - tag: soroban-dev-amd64 + tag: future-amd64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -53,7 +53,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: arch: arm64 - tag: soroban-dev-arm64 + tag: future-arm64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -74,5 +74,5 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag: soroban-dev + tag: future images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} From ee45a65d56b84e1c803a9fff844c42a460838f77 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:45:24 +1000 Subject: [PATCH 02/14] Push images with version numbers in labels --- .github/workflows/build.yml | 43 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 31bcfc6a..a4a9f144 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -380,10 +380,25 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + calc-push-release-version: + # Calculate the version of the release. + # TODO: Uncomment: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + outputs: + count: ${{ steps.count.outputs.count }} + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ env.HEAD_SHA }} + - id: count + run: + count="$(git rev-list HEAD --count --first-parent)" + echo "count=$count" >> $GITHUB_OUTPUT + push-release: # Push image to registry after test for master. - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - needs: [build, test] + # TODO: Uncomment: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + needs: [build, test, calc-push-release-version] permissions: packages: write statuses: write @@ -392,14 +407,16 @@ jobs: - uses: actions/checkout@v2 with: ref: ${{ env.HEAD_SHA }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ env.HEAD_SHA }} - artifact_name: image-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + - run: + echo v${{ needs.calc-push-release-version.outputs.count }}-${{ env.IMAGE }} + # TODO: Uncomment: - id: push + # TODO: Uncomment: uses: ./.github/actions/push + # TODO: Uncomment: with: + # TODO: Uncomment: head_sha: ${{ env.HEAD_SHA }} + # TODO: Uncomment: artifact_name: image-${{ inputs.arch }} + # TODO: Uncomment: artifact_image_file: image + # TODO: Uncomment: arch: ${{ inputs.arch }} + # TODO: Uncomment: image: v${{ needs.calc-push-release-version.outputs.count }}-${{ env.IMAGE }} + # TODO: Uncomment: registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + # TODO: Uncomment: username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + # TODO: Uncomment: password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} From f2050e461bc0fec10b35590944357f031f3e1907 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:41:47 +1000 Subject: [PATCH 03/14] Give images pushed by CI version numbers --- .github/workflows/build-future.yml | 47 ++- .github/workflows/build-image.yml | 408 +++++++++++++++++++++++++ .github/workflows/build-latest.yml | 47 ++- .github/workflows/build-testing.yml | 47 ++- .github/workflows/build.yml | 459 ++++------------------------ 5 files changed, 527 insertions(+), 481 deletions(-) create mode 100644 .github/workflows/build-image.yml diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index ff5d0fcf..f006d23c 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -4,37 +4,33 @@ name: Future # software that have been informally released to the futurenet network. on: - push: - branches: - - master - pull_request: - -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# master, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' + sha: + description: 'Sha to build' + type: 'string' + required: true jobs: - complete: - if: always() - needs: [manifest] - runs-on: ubuntu-latest - steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 - amd64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: amd64 - tag: future-amd64 + tag: ${{ inputs.tag-prefix }}future-amd64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -47,13 +43,14 @@ jobs: } arm64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: arm64 - tag: future-arm64 + tag: ${{ inputs.tag-prefix }}future-arm64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -74,5 +71,5 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag: future + tag: ${{ inputs.tag-prefix }}future images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 00000000..16b23e33 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,408 @@ +on: + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + sha: + description: 'Sha to build' + type: 'string' + required: true + arch: + description: 'Architecture to build the image for (amd64, arm64)' + type: 'string' + required: true + tag: + description: 'Tag to use on the image name' + type: 'string' + required: true + core_repo: + description: 'Git repo for stellar-core' + type: 'string' + default: 'https://github.com/stellar/stellar-core.git' + core_ref: + description: 'Git ref for the stellar-core repo' + type: 'string' + required: true + core_configure_flags: + description: 'CONFIGURE_FLAGS used when building stellar-core' + type: 'string' + default: '--disable-tests' + core_supports_enable_soroban_diagnostic_events: + description: 'Indicator whether stellar-core supports the ENABLE_SOROBAN_DIAGNOSTIC_EVENTS config' + type: 'string' + default: 'false' + core_build_runner_type: + description: 'The GitHub Runner instance type to build stellar-core on' + type: 'string' + default: 'ubuntu-latest' + go_ref: + description: 'Git ref for the stellar/go repo (stellar-horizon, stellar-friendbot)' + type: 'string' + required: true + xdr_ref: + description: 'Git ref for the stellar/rs-stellar-xdr repo' + type: 'string' + required: false + soroban_tools_ref: + description: 'Git ref for the stellar/soroban-tools repo (soroban-rpc)' + type: 'string' + required: true + soroban_rpc_build_runner_type: + description: 'The GitHub Runner instance type to build soroban-rpc on' + type: 'string' + default: 'ubuntu-latest' + test_matrix: + description: 'JSON matrix for the test job' + type: 'string' + required: true + outputs: + image: + description: 'Image pushed as a result of this build' + value: ${{ jobs.build.outputs.image }} + +env: + IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} + GO_REPO_BRANCH: ${{ inputs.go_ref }} + SOROBAN_TOOLS_REPO_BRANCH: ${{ inputs.soroban_tools_ref }} + CORE_REPO: ${{ inputs.core_repo }} + CORE_REPO_REF: ${{ inputs.core_ref }} + XDR_REPO_REF: ${{ inputs.xdr_ref }} + +jobs: + + build-stellar-core: + runs-on: ${{ inputs.core_build_runner_type }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Core Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f docker/Dockerfile.testing -t stellar-core:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + ${{ env.CORE_REPO }}#${{ env.CORE_REPO_REF }} + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' + - name: Upload Stellar-Core Image + uses: actions/upload-artifact@v2 + with: + name: image-stellar-core-${{ inputs.arch }} + path: /tmp/image + +# don't use caches on horizon builds as the git ref for it can be a branch name that refers to +# different commits over time, cache key won't invalidate correctly. + build-stellar-horizon: + runs-on: ubuntu-latest + steps: + - name: Checkout Quickstart for Horizon docker file + uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Setup buildx + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Horizon Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f Dockerfile.horizon --target builder + -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image + --build-arg REF="${{ env.GO_REPO_BRANCH }}" . + - name: Upload Stellar-Horizon Image + uses: actions/upload-artifact@v2 + with: + name: image-stellar-horizon-${{ inputs.arch }} + path: /tmp/image + +# don't use caches on friendbot builds as the git ref for it can be a branch name that refers to +# different commits over time, cache key won't invalidate correctly. + build-stellar-friendbot: + runs-on: ubuntu-latest + steps: + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Setup buildx + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Friendbot Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f services/friendbot/docker/Dockerfile -t stellar-friendbot:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + https://github.com/stellar/go.git#${{ env.GO_REPO_BRANCH }} + - name: Upload Stellar-Friendbot Image + uses: actions/upload-artifact@v2 + with: + name: image-stellar-friendbot-${{ inputs.arch }} + path: /tmp/image + + build-stellar-soroban-rpc: + runs-on: ${{ inputs.soroban_rpc_build_runner_type }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_TOOLS_REPO_BRANCH }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Soroban-Rpc Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f cmd/soroban-rpc/docker/Dockerfile --target build + -t stellar-soroban-rpc:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + https://github.com/stellar/soroban-tools.git#${{ env.SOROBAN_TOOLS_REPO_BRANCH }} + - name: Upload Stellar-Soroban-Rpc Image + uses: actions/upload-artifact@v2 + with: + name: image-stellar-soroban-rpc-${{ inputs.arch }} + path: /tmp/image + + build-rs-stellar-xdr: + runs-on: ubuntu-latest + steps: + - name: Checkout Quickstart for Horizon docker file + uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Rs-Xdr Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f Dockerfile.xdr --target builder + -t stellar-rs-xdr:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg REPO=https://github.com/stellar/rs-stellar-xdr.git + --build-arg REF="${{ env.XDR_REPO_REF }}" . + - name: Upload Stellar-Rs-Xdr Image + uses: actions/upload-artifact@v2 + with: + name: image-rs-stellar-xdr-${{ inputs.arch }} + path: /tmp/image + + build: + needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] + outputs: + image: ${{ steps.image.outputs.name }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - name: Download Stellar XDR + uses: actions/download-artifact@v2 + with: + name: image-rs-stellar-xdr-${{ inputs.arch }} + path: /tmp/stellar-xdr + - name: Download Stellar-Core Image + uses: actions/download-artifact@v2 + with: + name: image-stellar-core-${{ inputs.arch }} + path: /tmp/stellar-core + - name: Download Stellar-Horizon Image + uses: actions/download-artifact@v2 + with: + name: image-stellar-horizon-${{ inputs.arch }} + path: /tmp/stellar-horizon + - name: Download Stellar-Friendbot Image + uses: actions/download-artifact@v2 + with: + name: image-stellar-friendbot-${{ inputs.arch }} + path: /tmp/stellar-friendbot + - name: Download Stellar-Soroban-Rpc Image + uses: actions/download-artifact@v2 + with: + name: image-stellar-soroban-rpc-${{ inputs.arch }} + path: /tmp/stellar-soroban-rpc + - name: Load Stellar-Core Image + run: docker load -i /tmp/stellar-core/image + - name: Load Stellar-Horizon Image + run: docker load -i /tmp/stellar-horizon/image + - name: Load Stellar-Friendbot Image + run: docker load -i /tmp/stellar-friendbot/image + - name: Load Stellar-Soroban-Rpc Image + run: docker load -i /tmp/stellar-soroban-rpc/image + - name: Load Stellar-Rs-Xdr Image + run: docker load -i /tmp/stellar-xdr/image + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - id: image + name: Image Name + run: echo "name=$IMAGE" >> $GITHUB_OUTPUT + - name: Pull Base Image + run: docker pull --platform linux/${{ inputs.arch }} ubuntu:22.04 + # Docker buildx cannot be used to build the dev quickstart image because + # buildx does not yet support importing existing images, like the core and + # horizon images above, into a buildx builder's cache. Buildx would be + # preferred because it can output a smaller image file faster than docker + # save can. Once buildx supports it we can update. + # https://github.com/docker/buildx/issues/847 + - name: Build Quickstart Image + run: > + docker build + --platform linux/${{ inputs.arch }} + -f Dockerfile + -t $IMAGE + --label org.opencontainers.image.revision="${{ inputs.sha }}" + . + --build-arg REVISION="${{ inputs.sha }}" + --build-arg STELLAR_XDR_IMAGE_REF=stellar-rs-xdr:${{ inputs.arch }} + --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:${{ inputs.arch }} + --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ inputs.arch }} + --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ inputs.arch }} + --build-arg SOROBAN_RPC_IMAGE_REF=stellar-soroban-rpc:${{ inputs.arch }} + --build-arg CORE_SUPPORTS_ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=${{ inputs.core_supports_enable_soroban_diagnostic_events }} + - name: Save Quickstart Image + run: docker save $IMAGE -o /tmp/image + - name: Upload Quickstart Image + uses: actions/upload-artifact@v2 + with: + name: image-${{ inputs.arch }} + path: /tmp/image + + test: + needs: build + strategy: + matrix: ${{ fromJSON(inputs.test_matrix) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - name: Download Quickstart Image + uses: actions/download-artifact@v2 + with: + name: image-${{ inputs.arch }} + path: /tmp/ + - name: Load Quickstart Image + run: docker load -i /tmp/image + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Run Quickstart Image + run: docker run --platform linux/${{ inputs.arch }} --rm -d -p "8000:8000" -p "11626:11626" --name stellar $IMAGE --${{ matrix.network }} ${{ matrix.options }} + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ^1 + - name: Sleep until supervisor is up + run: sleep 10 + - name: Run core test + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & + go run test_core.go + curl http://localhost:11626/info + - name: Run horizon up test + run: | + docker logs stellar -f & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_horizon_up.go + curl http://localhost:8000 + - name: Run horizon ingesting test + if: ${{ matrix.network != 'pubnet' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_horizon_ingesting.go + curl http://localhost:8000 + - name: Run friendbot test + if: ${{ matrix.network == 'local' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_friendbot.go + - name: Run soroban rpc test + if: ${{ contains(matrix.options, '--enable-soroban-rpc') }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f soroban-rpc" | docker exec -i stellar sh & + go run test_soroban_rpc_up.go + + push-pr: + # Push image to registry after build for pull requests from a local branch. + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + needs: build + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - id: push + uses: ./.github/actions/push + with: + head_sha: ${{ inputs.sha }} + artifact_name: image-${{ inputs.arch }} + artifact_image_file: image + arch: ${{ inputs.arch }} + image: ${{ env.IMAGE }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + + push-release: + # Push image to registry after test for master. + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + needs: [build, test] + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - id: push + uses: ./.github/actions/push + with: + head_sha: ${{ inputs.sha }} + artifact_name: image-${{ inputs.arch }} + artifact_image_file: image + arch: ${{ inputs.arch }} + image: ${{ env.IMAGE }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index c9fa51b3..a66c1916 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -5,37 +5,33 @@ name: Latest # tag for software that's been released which is not a release candidate. on: - push: - branches: - - master - pull_request: - -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# master, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' + sha: + description: 'Sha to build' + type: 'string' + required: true jobs: - complete: - if: always() - needs: [manifest] - runs-on: ubuntu-latest - steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 - amd64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: amd64 - tag: latest-amd64 + tag: ${{ inputs.tag-prefix }}latest-amd64 xdr_ref: v20.0.2 core_ref: v20.0.2 go_ref: horizon-v2.27.0 @@ -50,13 +46,14 @@ jobs: } arm64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: arm64 - tag: latest-arm64 + tag: ${{ inputs.tag-prefix }}latest-arm64 xdr_ref: v20.0.2 core_ref: v20.0.2 core_build_runner_type: ubuntu-latest-16-cores @@ -79,5 +76,5 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag: latest + tag: ${{ inputs.tag-prefix }}latest images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index 065f75d9..3815c17c 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -6,37 +6,33 @@ name: Testing # release. on: - push: - branches: - - master - pull_request: - -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# master, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' + sha: + description: 'Sha to build' + type: 'string' + required: true jobs: - complete: - if: always() - needs: [manifest] - runs-on: ubuntu-latest - steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 - amd64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: amd64 - tag: testing-amd64 + tag: ${{ inputs.tag-prefix }}testing-amd64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -52,13 +48,14 @@ jobs: } arm64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: arm64 - tag: testing-arm64 + tag: ${{ inputs.tag-prefix }}testing-arm64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -82,5 +79,5 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag: testing + tag: ${{ inputs.tag-prefix }}testing images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4a9f144..ccdf7587 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,422 +1,69 @@ +name: Build + on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - arch: - description: 'Architecture to build the image for (amd64, arm64)' - type: 'string' - required: true - tag: - description: 'Tag to use on the image name' - type: 'string' - required: true - core_repo: - description: 'Git repo for stellar-core' - type: 'string' - default: 'https://github.com/stellar/stellar-core.git' - core_ref: - description: 'Git ref for the stellar-core repo' - type: 'string' - required: true - core_configure_flags: - description: 'CONFIGURE_FLAGS used when building stellar-core' - type: 'string' - default: '--disable-tests' - core_supports_enable_soroban_diagnostic_events: - description: 'Indicator whether stellar-core supports the ENABLE_SOROBAN_DIAGNOSTIC_EVENTS config' - type: 'string' - default: 'false' - core_build_runner_type: - description: 'The GitHub Runner instance type to build stellar-core on' - type: 'string' - default: 'ubuntu-latest' - go_ref: - description: 'Git ref for the stellar/go repo (stellar-horizon, stellar-friendbot)' - type: 'string' - required: true - xdr_ref: - description: 'Git ref for the stellar/rs-stellar-xdr repo' - type: 'string' - required: false - soroban_tools_ref: - description: 'Git ref for the stellar/soroban-tools repo (soroban-rpc)' - type: 'string' - required: true - soroban_rpc_build_runner_type: - description: 'The GitHub Runner instance type to build soroban-rpc on' - type: 'string' - default: 'ubuntu-latest' - test_matrix: - description: 'JSON matrix for the test job' - type: 'string' - required: true - outputs: - image: - description: 'Image pushed as a result of this build' - value: ${{ jobs.build.outputs.image }} + push: + branches: + - master + pull_request: -env: - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} - GO_REPO_BRANCH: ${{ inputs.go_ref }} - SOROBAN_TOOLS_REPO_BRANCH: ${{ inputs.soroban_tools_ref }} - CORE_REPO: ${{ inputs.core_repo }} - CORE_REPO_REF: ${{ inputs.core_ref }} - XDR_REPO_REF: ${{ inputs.xdr_ref }} +# Prevent more than one build of this workflow for a branch to be running at the +# same time, and if multiple are queued, only run the latest, cancelling any +# already running build. The exception being any protected branch, such as +# master, where a build for every commit will run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true jobs: - build-stellar-core: - runs-on: ${{ inputs.core_build_runner_type }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Core Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f docker/Dockerfile.testing -t stellar-core:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - ${{ env.CORE_REPO }}#${{ env.CORE_REPO_REF }} - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' - - name: Upload Stellar-Core Image - uses: actions/upload-artifact@v2 - with: - name: image-stellar-core-${{ inputs.arch }} - path: /tmp/image - -# don't use caches on horizon builds as the git ref for it can be a branch name that refers to -# different commits over time, cache key won't invalidate correctly. - build-stellar-horizon: + complete: + if: always() + needs: [latest, testing, future] runs-on: ubuntu-latest steps: - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ env.HEAD_SHA }} - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Horizon Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.horizon --target builder - -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image - --build-arg REF="${{ env.GO_REPO_BRANCH }}" . - - name: Upload Stellar-Horizon Image - uses: actions/upload-artifact@v2 - with: - name: image-stellar-horizon-${{ inputs.arch }} - path: /tmp/image - -# don't use caches on friendbot builds as the git ref for it can be a branch name that refers to -# different commits over time, cache key won't invalidate correctly. - build-stellar-friendbot: - runs-on: ubuntu-latest - steps: - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Friendbot Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f services/friendbot/docker/Dockerfile -t stellar-friendbot:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - https://github.com/stellar/go.git#${{ env.GO_REPO_BRANCH }} - - name: Upload Stellar-Friendbot Image - uses: actions/upload-artifact@v2 - with: - name: image-stellar-friendbot-${{ inputs.arch }} - path: /tmp/image - - build-stellar-soroban-rpc: - runs-on: ${{ inputs.soroban_rpc_build_runner_type }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_TOOLS_REPO_BRANCH }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Soroban-Rpc Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f cmd/soroban-rpc/docker/Dockerfile --target build - -t stellar-soroban-rpc:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - https://github.com/stellar/soroban-tools.git#${{ env.SOROBAN_TOOLS_REPO_BRANCH }} - - name: Upload Stellar-Soroban-Rpc Image - uses: actions/upload-artifact@v2 - with: - name: image-stellar-soroban-rpc-${{ inputs.arch }} - path: /tmp/image + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 - build-rs-stellar-xdr: + tag-prefix: + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} runs-on: ubuntu-latest - steps: - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ env.HEAD_SHA }} - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Rs-Xdr Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.xdr --target builder - -t stellar-rs-xdr:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REPO=https://github.com/stellar/rs-stellar-xdr.git - --build-arg REF="${{ env.XDR_REPO_REF }}" . - - name: Upload Stellar-Rs-Xdr Image - uses: actions/upload-artifact@v2 - with: - name: image-rs-stellar-xdr-${{ inputs.arch }} - path: /tmp/image - - build: - needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] outputs: - image: ${{ steps.image.outputs.name }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ env.HEAD_SHA }} - - name: Download Stellar XDR - uses: actions/download-artifact@v2 - with: - name: image-rs-stellar-xdr-${{ inputs.arch }} - path: /tmp/stellar-xdr - - name: Download Stellar-Core Image - uses: actions/download-artifact@v2 - with: - name: image-stellar-core-${{ inputs.arch }} - path: /tmp/stellar-core - - name: Download Stellar-Horizon Image - uses: actions/download-artifact@v2 - with: - name: image-stellar-horizon-${{ inputs.arch }} - path: /tmp/stellar-horizon - - name: Download Stellar-Friendbot Image - uses: actions/download-artifact@v2 - with: - name: image-stellar-friendbot-${{ inputs.arch }} - path: /tmp/stellar-friendbot - - name: Download Stellar-Soroban-Rpc Image - uses: actions/download-artifact@v2 - with: - name: image-stellar-soroban-rpc-${{ inputs.arch }} - path: /tmp/stellar-soroban-rpc - - name: Load Stellar-Core Image - run: docker load -i /tmp/stellar-core/image - - name: Load Stellar-Horizon Image - run: docker load -i /tmp/stellar-horizon/image - - name: Load Stellar-Friendbot Image - run: docker load -i /tmp/stellar-friendbot/image - - name: Load Stellar-Soroban-Rpc Image - run: docker load -i /tmp/stellar-soroban-rpc/image - - name: Load Stellar-Rs-Xdr Image - run: docker load -i /tmp/stellar-xdr/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - id: image - name: Image Name - run: echo "name=$IMAGE" >> $GITHUB_OUTPUT - - name: Pull Base Image - run: docker pull --platform linux/${{ inputs.arch }} ubuntu:22.04 - # Docker buildx cannot be used to build the dev quickstart image because - # buildx does not yet support importing existing images, like the core and - # horizon images above, into a buildx builder's cache. Buildx would be - # preferred because it can output a smaller image file faster than docker - # save can. Once buildx supports it we can update. - # https://github.com/docker/buildx/issues/847 - - name: Build Quickstart Image - run: > - docker build - --platform linux/${{ inputs.arch }} - -f Dockerfile - -t $IMAGE - --label org.opencontainers.image.revision="${{ env.HEAD_SHA }}" - . - --build-arg REVISION="${{ env.HEAD_SHA }}" - --build-arg STELLAR_XDR_IMAGE_REF=stellar-rs-xdr:${{ inputs.arch }} - --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:${{ inputs.arch }} - --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ inputs.arch }} - --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ inputs.arch }} - --build-arg SOROBAN_RPC_IMAGE_REF=stellar-soroban-rpc:${{ inputs.arch }} - --build-arg CORE_SUPPORTS_ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=${{ inputs.core_supports_enable_soroban_diagnostic_events }} - - name: Save Quickstart Image - run: docker save $IMAGE -o /tmp/image - - name: Upload Quickstart Image - uses: actions/upload-artifact@v2 - with: - name: image-${{ inputs.arch }} - path: /tmp/image - - test: - needs: build - strategy: - matrix: ${{ fromJSON(inputs.test_matrix) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ env.HEAD_SHA }} - - name: Download Quickstart Image - uses: actions/download-artifact@v2 - with: - name: image-${{ inputs.arch }} - path: /tmp/ - - name: Load Quickstart Image - run: docker load -i /tmp/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Run Quickstart Image - run: docker run --platform linux/${{ inputs.arch }} --rm -d -p "8000:8000" -p "11626:11626" --name stellar $IMAGE --${{ matrix.network }} ${{ matrix.options }} - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ^1 - - name: Sleep until supervisor is up - run: sleep 10 - - name: Run core test - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & - go run test_core.go - curl http://localhost:11626/info - - name: Run horizon up test - run: | - docker logs stellar -f & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_horizon_up.go - curl http://localhost:8000 - - name: Run horizon ingesting test - if: ${{ matrix.network != 'pubnet' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_horizon_ingesting.go - curl http://localhost:8000 - - name: Run friendbot test - if: ${{ matrix.network == 'local' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_friendbot.go - - name: Run soroban rpc test - if: ${{ contains(matrix.options, '--enable-soroban-rpc') }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f soroban-rpc" | docker exec -i stellar sh & - go run test_soroban_rpc_up.go - - push-pr: - # Push image to registry after build for pull requests from a local branch. - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} - needs: build - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest + tag-prefix: ${{ steps.release.outputs.tag-prefix }} steps: - uses: actions/checkout@v2 with: - ref: ${{ env.HEAD_SHA }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ env.HEAD_SHA }} - artifact_name: image-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - calc-push-release-version: - # Calculate the version of the release. - # TODO: Uncomment: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - runs-on: ubuntu-latest - outputs: - count: ${{ steps.count.outputs.count }} - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ env.HEAD_SHA }} - - id: count + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - id: release run: count="$(git rev-list HEAD --count --first-parent)" - echo "count=$count" >> $GITHUB_OUTPUT + echo "tag-prefix=v${count}-" >> $GITHUB_OUTPUT - push-release: - # Push image to registry after test for master. - # TODO: Uncomment: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - needs: [build, test, calc-push-release-version] - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ env.HEAD_SHA }} - - run: - echo v${{ needs.calc-push-release-version.outputs.count }}-${{ env.IMAGE }} - # TODO: Uncomment: - id: push - # TODO: Uncomment: uses: ./.github/actions/push - # TODO: Uncomment: with: - # TODO: Uncomment: head_sha: ${{ env.HEAD_SHA }} - # TODO: Uncomment: artifact_name: image-${{ inputs.arch }} - # TODO: Uncomment: artifact_image_file: image - # TODO: Uncomment: arch: ${{ inputs.arch }} - # TODO: Uncomment: image: v${{ needs.calc-push-release-version.outputs.count }}-${{ env.IMAGE }} - # TODO: Uncomment: registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - # TODO: Uncomment: username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - # TODO: Uncomment: password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + latest: + needs: [tag-prefix] + uses: ./.github/workflows/build-latest.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} + + testing: + needs: [tag-prefix] + uses: ./.github/workflows/build-testing.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} + + future: + needs: [tag-prefix] + uses: ./.github/workflows/build-future.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} From 17b6957d9a3331e80f51cb41f669c8795b6909c0 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:43:50 +1000 Subject: [PATCH 04/14] fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ccdf7587..bd600de4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,6 @@ jobs: run: exit 1 tag-prefix: - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} runs-on: ubuntu-latest outputs: tag-prefix: ${{ steps.release.outputs.tag-prefix }} @@ -34,6 +33,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - id: release + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} run: count="$(git rev-list HEAD --count --first-parent)" echo "tag-prefix=v${count}-" >> $GITHUB_OUTPUT From 3f3d4eb3e33fbd62126efd2dc04683d53c905d6b Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 21:36:46 +1000 Subject: [PATCH 05/14] rename --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd600de4..3b29e3f5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: exit 1 - tag-prefix: + setup: runs-on: ubuntu-latest outputs: tag-prefix: ${{ steps.release.outputs.tag-prefix }} @@ -39,31 +39,31 @@ jobs: echo "tag-prefix=v${count}-" >> $GITHUB_OUTPUT latest: - needs: [tag-prefix] + needs: [setup] uses: ./.github/workflows/build-latest.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} sha: ${{ github.event.pull_request.head.sha || github.sha }} testing: - needs: [tag-prefix] + needs: [setup] uses: ./.github/workflows/build-testing.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} sha: ${{ github.event.pull_request.head.sha || github.sha }} future: - needs: [tag-prefix] + needs: [setup] uses: ./.github/workflows/build-future.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} sha: ${{ github.event.pull_request.head.sha || github.sha }} From e915e84079da4ac4efbf796a4f540bef19416507 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 22:51:24 +1000 Subject: [PATCH 06/14] upd --- .github/workflows/build-future.yml | 8 ++++---- .github/workflows/build-latest.yml | 8 ++++---- .github/workflows/build-testing.yml | 8 ++++---- .github/workflows/build.yml | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index f006d23c..2057ade9 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -11,14 +11,14 @@ on: DOCKERHUB_TOKEN: required: false inputs: - tag-prefix: - description: 'Prefix for the tag name' - type: 'string' - default: '' sha: description: 'Sha to build' type: 'string' required: true + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' jobs: diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index a66c1916..f4f23ef5 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -12,14 +12,14 @@ on: DOCKERHUB_TOKEN: required: false inputs: - tag-prefix: - description: 'Prefix for the tag name' - type: 'string' - default: '' sha: description: 'Sha to build' type: 'string' required: true + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' jobs: diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index 3815c17c..c763a6e8 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -13,14 +13,14 @@ on: DOCKERHUB_TOKEN: required: false inputs: - tag-prefix: - description: 'Prefix for the tag name' - type: 'string' - default: '' sha: description: 'Sha to build' type: 'string' required: true + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' jobs: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3b29e3f5..3aef954b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,8 +45,8 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} sha: ${{ github.event.pull_request.head.sha || github.sha }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} testing: needs: [setup] @@ -55,8 +55,8 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} sha: ${{ github.event.pull_request.head.sha || github.sha }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} future: needs: [setup] @@ -65,5 +65,5 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} sha: ${{ github.event.pull_request.head.sha || github.sha }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} From 9598f691b8798aa2b287bb645e73a29ce599d0e6 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 22:56:50 +1000 Subject: [PATCH 07/14] upd --- .github/workflows/build-image.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 16b23e33..adb8af27 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -97,7 +97,7 @@ jobs: --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' - name: Upload Stellar-Core Image - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: image-stellar-core-${{ inputs.arch }} path: /tmp/image @@ -124,7 +124,7 @@ jobs: -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image --build-arg REF="${{ env.GO_REPO_BRANCH }}" . - name: Upload Stellar-Horizon Image - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: image-stellar-horizon-${{ inputs.arch }} path: /tmp/image @@ -148,7 +148,7 @@ jobs: --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true https://github.com/stellar/go.git#${{ env.GO_REPO_BRANCH }} - name: Upload Stellar-Friendbot Image - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: image-stellar-friendbot-${{ inputs.arch }} path: /tmp/image @@ -177,7 +177,7 @@ jobs: --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true https://github.com/stellar/soroban-tools.git#${{ env.SOROBAN_TOOLS_REPO_BRANCH }} - name: Upload Stellar-Soroban-Rpc Image - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: image-stellar-soroban-rpc-${{ inputs.arch }} path: /tmp/image @@ -210,7 +210,7 @@ jobs: --build-arg REPO=https://github.com/stellar/rs-stellar-xdr.git --build-arg REF="${{ env.XDR_REPO_REF }}" . - name: Upload Stellar-Rs-Xdr Image - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: image-rs-stellar-xdr-${{ inputs.arch }} path: /tmp/image @@ -292,7 +292,7 @@ jobs: - name: Save Quickstart Image run: docker save $IMAGE -o /tmp/image - name: Upload Quickstart Image - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: image-${{ inputs.arch }} path: /tmp/image From 1dbc25efe6d41efbb9e393e036c91982ab06fc85 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 23:04:42 +1000 Subject: [PATCH 08/14] fix --- .github/actions/push/action.yml | 2 +- .github/workflows/build-image.yml | 40 +++++++++++++++---------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml index 9298ecf5..20937e64 100644 --- a/.github/actions/push/action.yml +++ b/.github/actions/push/action.yml @@ -26,7 +26,7 @@ runs: using: "composite" steps: - - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: ${{ inputs.artifact_name }} path: /tmp/ diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index adb8af27..ec132eb0 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -99,7 +99,7 @@ jobs: - name: Upload Stellar-Core Image uses: actions/upload-artifact@v4 with: - name: image-stellar-core-${{ inputs.arch }} + name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image # don't use caches on horizon builds as the git ref for it can be a branch name that refers to @@ -126,7 +126,7 @@ jobs: - name: Upload Stellar-Horizon Image uses: actions/upload-artifact@v4 with: - name: image-stellar-horizon-${{ inputs.arch }} + name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image # don't use caches on friendbot builds as the git ref for it can be a branch name that refers to @@ -150,7 +150,7 @@ jobs: - name: Upload Stellar-Friendbot Image uses: actions/upload-artifact@v4 with: - name: image-stellar-friendbot-${{ inputs.arch }} + name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image build-stellar-soroban-rpc: @@ -179,7 +179,7 @@ jobs: - name: Upload Stellar-Soroban-Rpc Image uses: actions/upload-artifact@v4 with: - name: image-stellar-soroban-rpc-${{ inputs.arch }} + name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image build-rs-stellar-xdr: @@ -212,7 +212,7 @@ jobs: - name: Upload Stellar-Rs-Xdr Image uses: actions/upload-artifact@v4 with: - name: image-rs-stellar-xdr-${{ inputs.arch }} + name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image build: @@ -225,29 +225,29 @@ jobs: with: ref: ${{ inputs.sha }} - name: Download Stellar XDR - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: image-rs-stellar-xdr-${{ inputs.arch }} + name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/stellar-xdr - name: Download Stellar-Core Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: image-stellar-core-${{ inputs.arch }} + name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/stellar-core - name: Download Stellar-Horizon Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: image-stellar-horizon-${{ inputs.arch }} + name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/stellar-horizon - name: Download Stellar-Friendbot Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: image-stellar-friendbot-${{ inputs.arch }} + name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/stellar-friendbot - name: Download Stellar-Soroban-Rpc Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: image-stellar-soroban-rpc-${{ inputs.arch }} + name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/stellar-soroban-rpc - name: Load Stellar-Core Image run: docker load -i /tmp/stellar-core/image @@ -294,7 +294,7 @@ jobs: - name: Upload Quickstart Image uses: actions/upload-artifact@v4 with: - name: image-${{ inputs.arch }} + name: image-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/image test: @@ -307,9 +307,9 @@ jobs: with: ref: ${{ inputs.sha }} - name: Download Quickstart Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: - name: image-${{ inputs.arch }} + name: image-${{ inputs.tag }}-${{ inputs.arch }} path: /tmp/ - name: Load Quickstart Image run: docker load -i /tmp/image @@ -375,7 +375,7 @@ jobs: uses: ./.github/actions/push with: head_sha: ${{ inputs.sha }} - artifact_name: image-${{ inputs.arch }} + artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} artifact_image_file: image arch: ${{ inputs.arch }} image: ${{ env.IMAGE }} @@ -399,7 +399,7 @@ jobs: uses: ./.github/actions/push with: head_sha: ${{ inputs.sha }} - artifact_name: image-${{ inputs.arch }} + artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} artifact_image_file: image arch: ${{ inputs.arch }} image: ${{ env.IMAGE }} From e59911f3367af9a4ca8e5eea181a714c73917bb7 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 23:19:22 +1000 Subject: [PATCH 09/14] aliasing --- .github/workflows/build-future.yml | 1 + .github/workflows/build-latest.yml | 1 + .github/workflows/build-testing.yml | 1 + .github/workflows/build.yml | 5 ++--- .github/workflows/manifest.yml | 7 +++++++ 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index 2057ade9..870811fc 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -72,4 +72,5 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: tag: ${{ inputs.tag-prefix }}future + tag-alias: future images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index f4f23ef5..e8995433 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -77,4 +77,5 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: tag: ${{ inputs.tag-prefix }}latest + tag-alias: latest images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index c763a6e8..4b50eaff 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -80,4 +80,5 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: tag: ${{ inputs.tag-prefix }}testing + tag-alias: testing images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3aef954b..296c90eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,13 +27,12 @@ jobs: setup: runs-on: ubuntu-latest outputs: - tag-prefix: ${{ steps.release.outputs.tag-prefix }} + tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} steps: - uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - - id: release - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + - id: tag-prefix run: count="$(git rev-list HEAD --count --first-parent)" echo "tag-prefix=v${count}-" >> $GITHUB_OUTPUT diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml index 7bfbca7a..34484d11 100644 --- a/.github/workflows/manifest.yml +++ b/.github/workflows/manifest.yml @@ -10,6 +10,10 @@ on: description: 'Tag to use as the manifest list image name' type: 'string' required: true + tag-alias: + description: 'Tag to alias to the tag of the manifest, e.g. "latest"' + type: 'string' + required: true images: description: 'Space separated list of images to include in the manifest list' type: 'string' @@ -22,6 +26,7 @@ on: env: HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} + IMAGE_ALIAS: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag-alias) || inputs.tag-alias) }} REGISTRY: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} jobs: @@ -53,6 +58,8 @@ jobs: docker manifest create ${{ env.IMAGE }} ${{ inputs.images }} - run: | docker manifest push ${{ env.IMAGE }} + - run: | + docker buildx imagetools create -t ${{ env.IMAGE_ALIAS }} ${{ env.IMAGE }} - uses: actions/github-script@v5 with: script: | From 4426be664ecef62c0f52a53c316edf468b93668a Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Mon, 22 Jan 2024 23:29:07 +1000 Subject: [PATCH 10/14] fix --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 296c90eb..4f033e2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - id: tag-prefix - run: + run: | count="$(git rev-list HEAD --count --first-parent)" echo "tag-prefix=v${count}-" >> $GITHUB_OUTPUT From f6c6b3999bbecd41b85614d288362ae8e8de87ba Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 23 Jan 2024 00:36:03 +1000 Subject: [PATCH 11/14] fix --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f033e2b..4c22b08a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,7 @@ jobs: steps: - uses: actions/checkout@v2 with: + fetch-depth: 0 # Get all history for the sha count below. ref: ${{ github.event.pull_request.head.sha || github.sha }} - id: tag-prefix run: | From 7471fbf2b8f76e183e0fe10f38548557767dd1d4 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 23 Jan 2024 01:07:18 +1000 Subject: [PATCH 12/14] consistent prefix on numbers in label --- .github/workflows/build-image.yml | 2 +- .github/workflows/manifest.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index ec132eb0..d951cebc 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -64,7 +64,7 @@ on: value: ${{ jobs.build.outputs.image }} env: - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} + IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} GO_REPO_BRANCH: ${{ inputs.go_ref }} SOROBAN_TOOLS_REPO_BRANCH: ${{ inputs.soroban_tools_ref }} CORE_REPO: ${{ inputs.core_repo }} diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml index 34484d11..6f4d86c7 100644 --- a/.github/workflows/manifest.yml +++ b/.github/workflows/manifest.yml @@ -25,8 +25,8 @@ on: env: HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} - IMAGE_ALIAS: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag-alias) || inputs.tag-alias) }} + IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} + IMAGE_ALIAS: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr{0}-{1}', github.event.pull_request.number, inputs.tag-alias) || inputs.tag-alias) }} REGISTRY: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} jobs: From 0cbcb20a7cc0209daa05859c746c423ab3218b7a Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:58:42 +1000 Subject: [PATCH 13/14] Reduce changes --- .github/workflows/build-image.yml | 408 --------------------------- .github/workflows/build-start.yml | 69 +++++ .github/workflows/build.yml | 445 ++++++++++++++++++++++++++---- 3 files changed, 461 insertions(+), 461 deletions(-) delete mode 100644 .github/workflows/build-image.yml create mode 100644 .github/workflows/build-start.yml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml deleted file mode 100644 index d951cebc..00000000 --- a/.github/workflows/build-image.yml +++ /dev/null @@ -1,408 +0,0 @@ -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - sha: - description: 'Sha to build' - type: 'string' - required: true - arch: - description: 'Architecture to build the image for (amd64, arm64)' - type: 'string' - required: true - tag: - description: 'Tag to use on the image name' - type: 'string' - required: true - core_repo: - description: 'Git repo for stellar-core' - type: 'string' - default: 'https://github.com/stellar/stellar-core.git' - core_ref: - description: 'Git ref for the stellar-core repo' - type: 'string' - required: true - core_configure_flags: - description: 'CONFIGURE_FLAGS used when building stellar-core' - type: 'string' - default: '--disable-tests' - core_supports_enable_soroban_diagnostic_events: - description: 'Indicator whether stellar-core supports the ENABLE_SOROBAN_DIAGNOSTIC_EVENTS config' - type: 'string' - default: 'false' - core_build_runner_type: - description: 'The GitHub Runner instance type to build stellar-core on' - type: 'string' - default: 'ubuntu-latest' - go_ref: - description: 'Git ref for the stellar/go repo (stellar-horizon, stellar-friendbot)' - type: 'string' - required: true - xdr_ref: - description: 'Git ref for the stellar/rs-stellar-xdr repo' - type: 'string' - required: false - soroban_tools_ref: - description: 'Git ref for the stellar/soroban-tools repo (soroban-rpc)' - type: 'string' - required: true - soroban_rpc_build_runner_type: - description: 'The GitHub Runner instance type to build soroban-rpc on' - type: 'string' - default: 'ubuntu-latest' - test_matrix: - description: 'JSON matrix for the test job' - type: 'string' - required: true - outputs: - image: - description: 'Image pushed as a result of this build' - value: ${{ jobs.build.outputs.image }} - -env: - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} - GO_REPO_BRANCH: ${{ inputs.go_ref }} - SOROBAN_TOOLS_REPO_BRANCH: ${{ inputs.soroban_tools_ref }} - CORE_REPO: ${{ inputs.core_repo }} - CORE_REPO_REF: ${{ inputs.core_ref }} - XDR_REPO_REF: ${{ inputs.xdr_ref }} - -jobs: - - build-stellar-core: - runs-on: ${{ inputs.core_build_runner_type }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Core Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f docker/Dockerfile.testing -t stellar-core:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - ${{ env.CORE_REPO }}#${{ env.CORE_REPO_REF }} - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' - - name: Upload Stellar-Core Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - -# don't use caches on horizon builds as the git ref for it can be a branch name that refers to -# different commits over time, cache key won't invalidate correctly. - build-stellar-horizon: - runs-on: ubuntu-latest - steps: - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Horizon Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.horizon --target builder - -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image - --build-arg REF="${{ env.GO_REPO_BRANCH }}" . - - name: Upload Stellar-Horizon Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - -# don't use caches on friendbot builds as the git ref for it can be a branch name that refers to -# different commits over time, cache key won't invalidate correctly. - build-stellar-friendbot: - runs-on: ubuntu-latest - steps: - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Friendbot Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f services/friendbot/docker/Dockerfile -t stellar-friendbot:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - https://github.com/stellar/go.git#${{ env.GO_REPO_BRANCH }} - - name: Upload Stellar-Friendbot Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - - build-stellar-soroban-rpc: - runs-on: ${{ inputs.soroban_rpc_build_runner_type }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_TOOLS_REPO_BRANCH }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Soroban-Rpc Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f cmd/soroban-rpc/docker/Dockerfile --target build - -t stellar-soroban-rpc:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - https://github.com/stellar/soroban-tools.git#${{ env.SOROBAN_TOOLS_REPO_BRANCH }} - - name: Upload Stellar-Soroban-Rpc Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - - build-rs-stellar-xdr: - runs-on: ubuntu-latest - steps: - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Rs-Xdr Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.xdr --target builder - -t stellar-rs-xdr:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REPO=https://github.com/stellar/rs-stellar-xdr.git - --build-arg REF="${{ env.XDR_REPO_REF }}" . - - name: Upload Stellar-Rs-Xdr Image - uses: actions/upload-artifact@v4 - with: - name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - - build: - needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] - outputs: - image: ${{ steps.image.outputs.name }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - name: Download Stellar XDR - uses: actions/download-artifact@v4 - with: - name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-xdr - - name: Download Stellar-Core Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-core - - name: Download Stellar-Horizon Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-horizon - - name: Download Stellar-Friendbot Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-friendbot - - name: Download Stellar-Soroban-Rpc Image - uses: actions/download-artifact@v4 - with: - name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-soroban-rpc - - name: Load Stellar-Core Image - run: docker load -i /tmp/stellar-core/image - - name: Load Stellar-Horizon Image - run: docker load -i /tmp/stellar-horizon/image - - name: Load Stellar-Friendbot Image - run: docker load -i /tmp/stellar-friendbot/image - - name: Load Stellar-Soroban-Rpc Image - run: docker load -i /tmp/stellar-soroban-rpc/image - - name: Load Stellar-Rs-Xdr Image - run: docker load -i /tmp/stellar-xdr/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - id: image - name: Image Name - run: echo "name=$IMAGE" >> $GITHUB_OUTPUT - - name: Pull Base Image - run: docker pull --platform linux/${{ inputs.arch }} ubuntu:22.04 - # Docker buildx cannot be used to build the dev quickstart image because - # buildx does not yet support importing existing images, like the core and - # horizon images above, into a buildx builder's cache. Buildx would be - # preferred because it can output a smaller image file faster than docker - # save can. Once buildx supports it we can update. - # https://github.com/docker/buildx/issues/847 - - name: Build Quickstart Image - run: > - docker build - --platform linux/${{ inputs.arch }} - -f Dockerfile - -t $IMAGE - --label org.opencontainers.image.revision="${{ inputs.sha }}" - . - --build-arg REVISION="${{ inputs.sha }}" - --build-arg STELLAR_XDR_IMAGE_REF=stellar-rs-xdr:${{ inputs.arch }} - --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:${{ inputs.arch }} - --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ inputs.arch }} - --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ inputs.arch }} - --build-arg SOROBAN_RPC_IMAGE_REF=stellar-soroban-rpc:${{ inputs.arch }} - --build-arg CORE_SUPPORTS_ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=${{ inputs.core_supports_enable_soroban_diagnostic_events }} - - name: Save Quickstart Image - run: docker save $IMAGE -o /tmp/image - - name: Upload Quickstart Image - uses: actions/upload-artifact@v4 - with: - name: image-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - - test: - needs: build - strategy: - matrix: ${{ fromJSON(inputs.test_matrix) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - - name: Download Quickstart Image - uses: actions/download-artifact@v4 - with: - name: image-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/ - - name: Load Quickstart Image - run: docker load -i /tmp/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Run Quickstart Image - run: docker run --platform linux/${{ inputs.arch }} --rm -d -p "8000:8000" -p "11626:11626" --name stellar $IMAGE --${{ matrix.network }} ${{ matrix.options }} - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ^1 - - name: Sleep until supervisor is up - run: sleep 10 - - name: Run core test - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & - go run test_core.go - curl http://localhost:11626/info - - name: Run horizon up test - run: | - docker logs stellar -f & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_horizon_up.go - curl http://localhost:8000 - - name: Run horizon ingesting test - if: ${{ matrix.network != 'pubnet' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_horizon_ingesting.go - curl http://localhost:8000 - - name: Run friendbot test - if: ${{ matrix.network == 'local' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_friendbot.go - - name: Run soroban rpc test - if: ${{ contains(matrix.options, '--enable-soroban-rpc') }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f soroban-rpc" | docker exec -i stellar sh & - go run test_soroban_rpc_up.go - - push-pr: - # Push image to registry after build for pull requests from a local branch. - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} - needs: build - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ inputs.sha }} - artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - push-release: - # Push image to registry after test for master. - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - needs: [build, test] - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ inputs.sha }} - artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} diff --git a/.github/workflows/build-start.yml b/.github/workflows/build-start.yml new file mode 100644 index 00000000..4c22b08a --- /dev/null +++ b/.github/workflows/build-start.yml @@ -0,0 +1,69 @@ +name: Build + +on: + push: + branches: + - master + pull_request: + +# Prevent more than one build of this workflow for a branch to be running at the +# same time, and if multiple are queued, only run the latest, cancelling any +# already running build. The exception being any protected branch, such as +# master, where a build for every commit will run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + +jobs: + + complete: + if: always() + needs: [latest, testing, future] + runs-on: ubuntu-latest + steps: + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 + + setup: + runs-on: ubuntu-latest + outputs: + tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Get all history for the sha count below. + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - id: tag-prefix + run: | + count="$(git rev-list HEAD --count --first-parent)" + echo "tag-prefix=v${count}-" >> $GITHUB_OUTPUT + + latest: + needs: [setup] + uses: ./.github/workflows/build-latest.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + sha: ${{ github.event.pull_request.head.sha || github.sha }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} + + testing: + needs: [setup] + uses: ./.github/workflows/build-testing.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + sha: ${{ github.event.pull_request.head.sha || github.sha }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} + + future: + needs: [setup] + uses: ./.github/workflows/build-future.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + sha: ${{ github.event.pull_request.head.sha || github.sha }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c22b08a..d951cebc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,69 +1,408 @@ -name: Build - on: - push: - branches: - - master - pull_request: + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + sha: + description: 'Sha to build' + type: 'string' + required: true + arch: + description: 'Architecture to build the image for (amd64, arm64)' + type: 'string' + required: true + tag: + description: 'Tag to use on the image name' + type: 'string' + required: true + core_repo: + description: 'Git repo for stellar-core' + type: 'string' + default: 'https://github.com/stellar/stellar-core.git' + core_ref: + description: 'Git ref for the stellar-core repo' + type: 'string' + required: true + core_configure_flags: + description: 'CONFIGURE_FLAGS used when building stellar-core' + type: 'string' + default: '--disable-tests' + core_supports_enable_soroban_diagnostic_events: + description: 'Indicator whether stellar-core supports the ENABLE_SOROBAN_DIAGNOSTIC_EVENTS config' + type: 'string' + default: 'false' + core_build_runner_type: + description: 'The GitHub Runner instance type to build stellar-core on' + type: 'string' + default: 'ubuntu-latest' + go_ref: + description: 'Git ref for the stellar/go repo (stellar-horizon, stellar-friendbot)' + type: 'string' + required: true + xdr_ref: + description: 'Git ref for the stellar/rs-stellar-xdr repo' + type: 'string' + required: false + soroban_tools_ref: + description: 'Git ref for the stellar/soroban-tools repo (soroban-rpc)' + type: 'string' + required: true + soroban_rpc_build_runner_type: + description: 'The GitHub Runner instance type to build soroban-rpc on' + type: 'string' + default: 'ubuntu-latest' + test_matrix: + description: 'JSON matrix for the test job' + type: 'string' + required: true + outputs: + image: + description: 'Image pushed as a result of this build' + value: ${{ jobs.build.outputs.image }} -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# master, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true +env: + IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} + GO_REPO_BRANCH: ${{ inputs.go_ref }} + SOROBAN_TOOLS_REPO_BRANCH: ${{ inputs.soroban_tools_ref }} + CORE_REPO: ${{ inputs.core_repo }} + CORE_REPO_REF: ${{ inputs.core_ref }} + XDR_REPO_REF: ${{ inputs.xdr_ref }} jobs: - complete: - if: always() - needs: [latest, testing, future] + build-stellar-core: + runs-on: ${{ inputs.core_build_runner_type }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Core Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f docker/Dockerfile.testing -t stellar-core:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + ${{ env.CORE_REPO }}#${{ env.CORE_REPO_REF }} + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' + - name: Upload Stellar-Core Image + uses: actions/upload-artifact@v4 + with: + name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image + +# don't use caches on horizon builds as the git ref for it can be a branch name that refers to +# different commits over time, cache key won't invalidate correctly. + build-stellar-horizon: runs-on: ubuntu-latest steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 + - name: Checkout Quickstart for Horizon docker file + uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Setup buildx + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Horizon Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f Dockerfile.horizon --target builder + -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image + --build-arg REF="${{ env.GO_REPO_BRANCH }}" . + - name: Upload Stellar-Horizon Image + uses: actions/upload-artifact@v4 + with: + name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image + +# don't use caches on friendbot builds as the git ref for it can be a branch name that refers to +# different commits over time, cache key won't invalidate correctly. + build-stellar-friendbot: + runs-on: ubuntu-latest + steps: + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Setup buildx + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Friendbot Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f services/friendbot/docker/Dockerfile -t stellar-friendbot:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + https://github.com/stellar/go.git#${{ env.GO_REPO_BRANCH }} + - name: Upload Stellar-Friendbot Image + uses: actions/upload-artifact@v4 + with: + name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image + + build-stellar-soroban-rpc: + runs-on: ${{ inputs.soroban_rpc_build_runner_type }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_TOOLS_REPO_BRANCH }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Soroban-Rpc Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f cmd/soroban-rpc/docker/Dockerfile --target build + -t stellar-soroban-rpc:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + https://github.com/stellar/soroban-tools.git#${{ env.SOROBAN_TOOLS_REPO_BRANCH }} + - name: Upload Stellar-Soroban-Rpc Image + uses: actions/upload-artifact@v4 + with: + name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image - setup: + build-rs-stellar-xdr: runs-on: ubuntu-latest + steps: + - name: Checkout Quickstart for Horizon docker file + uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Rs-Xdr Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f Dockerfile.xdr --target builder + -t stellar-rs-xdr:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg REPO=https://github.com/stellar/rs-stellar-xdr.git + --build-arg REF="${{ env.XDR_REPO_REF }}" . + - name: Upload Stellar-Rs-Xdr Image + uses: actions/upload-artifact@v4 + with: + name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image + + build: + needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] outputs: - tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} + image: ${{ steps.image.outputs.name }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - name: Download Stellar XDR + uses: actions/download-artifact@v4 + with: + name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/stellar-xdr + - name: Download Stellar-Core Image + uses: actions/download-artifact@v4 + with: + name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/stellar-core + - name: Download Stellar-Horizon Image + uses: actions/download-artifact@v4 + with: + name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/stellar-horizon + - name: Download Stellar-Friendbot Image + uses: actions/download-artifact@v4 + with: + name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/stellar-friendbot + - name: Download Stellar-Soroban-Rpc Image + uses: actions/download-artifact@v4 + with: + name: image-stellar-soroban-rpc-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/stellar-soroban-rpc + - name: Load Stellar-Core Image + run: docker load -i /tmp/stellar-core/image + - name: Load Stellar-Horizon Image + run: docker load -i /tmp/stellar-horizon/image + - name: Load Stellar-Friendbot Image + run: docker load -i /tmp/stellar-friendbot/image + - name: Load Stellar-Soroban-Rpc Image + run: docker load -i /tmp/stellar-soroban-rpc/image + - name: Load Stellar-Rs-Xdr Image + run: docker load -i /tmp/stellar-xdr/image + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - id: image + name: Image Name + run: echo "name=$IMAGE" >> $GITHUB_OUTPUT + - name: Pull Base Image + run: docker pull --platform linux/${{ inputs.arch }} ubuntu:22.04 + # Docker buildx cannot be used to build the dev quickstart image because + # buildx does not yet support importing existing images, like the core and + # horizon images above, into a buildx builder's cache. Buildx would be + # preferred because it can output a smaller image file faster than docker + # save can. Once buildx supports it we can update. + # https://github.com/docker/buildx/issues/847 + - name: Build Quickstart Image + run: > + docker build + --platform linux/${{ inputs.arch }} + -f Dockerfile + -t $IMAGE + --label org.opencontainers.image.revision="${{ inputs.sha }}" + . + --build-arg REVISION="${{ inputs.sha }}" + --build-arg STELLAR_XDR_IMAGE_REF=stellar-rs-xdr:${{ inputs.arch }} + --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:${{ inputs.arch }} + --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ inputs.arch }} + --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ inputs.arch }} + --build-arg SOROBAN_RPC_IMAGE_REF=stellar-soroban-rpc:${{ inputs.arch }} + --build-arg CORE_SUPPORTS_ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=${{ inputs.core_supports_enable_soroban_diagnostic_events }} + - name: Save Quickstart Image + run: docker save $IMAGE -o /tmp/image + - name: Upload Quickstart Image + uses: actions/upload-artifact@v4 + with: + name: image-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/image + + test: + needs: build + strategy: + matrix: ${{ fromJSON(inputs.test_matrix) }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: - fetch-depth: 0 # Get all history for the sha count below. - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - id: tag-prefix + ref: ${{ inputs.sha }} + - name: Download Quickstart Image + uses: actions/download-artifact@v4 + with: + name: image-${{ inputs.tag }}-${{ inputs.arch }} + path: /tmp/ + - name: Load Quickstart Image + run: docker load -i /tmp/image + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Run Quickstart Image + run: docker run --platform linux/${{ inputs.arch }} --rm -d -p "8000:8000" -p "11626:11626" --name stellar $IMAGE --${{ matrix.network }} ${{ matrix.options }} + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ^1 + - name: Sleep until supervisor is up + run: sleep 10 + - name: Run core test run: | - count="$(git rev-list HEAD --count --first-parent)" - echo "tag-prefix=v${count}-" >> $GITHUB_OUTPUT - - latest: - needs: [setup] - uses: ./.github/workflows/build-latest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} + docker logs stellar -f & + echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & + go run test_core.go + curl http://localhost:11626/info + - name: Run horizon up test + run: | + docker logs stellar -f & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_horizon_up.go + curl http://localhost:8000 + - name: Run horizon ingesting test + if: ${{ matrix.network != 'pubnet' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_horizon_ingesting.go + curl http://localhost:8000 + - name: Run friendbot test + if: ${{ matrix.network == 'local' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_friendbot.go + - name: Run soroban rpc test + if: ${{ contains(matrix.options, '--enable-soroban-rpc') }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f soroban-rpc" | docker exec -i stellar sh & + go run test_soroban_rpc_up.go - testing: - needs: [setup] - uses: ./.github/workflows/build-testing.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} + push-pr: + # Push image to registry after build for pull requests from a local branch. + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + needs: build + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - id: push + uses: ./.github/actions/push + with: + head_sha: ${{ inputs.sha }} + artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} + artifact_image_file: image + arch: ${{ inputs.arch }} + image: ${{ env.IMAGE }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - future: - needs: [setup] - uses: ./.github/workflows/build-future.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} + push-release: + # Push image to registry after test for master. + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + needs: [build, test] + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - id: push + uses: ./.github/actions/push + with: + head_sha: ${{ inputs.sha }} + artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} + artifact_image_file: image + arch: ${{ inputs.arch }} + image: ${{ env.IMAGE }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} From 56a2108321af34cd12691ad46d6696cc41db7fc8 Mon Sep 17 00:00:00 2001 From: Leigh McCulloch <351529+leighmcculloch@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:00:07 +1000 Subject: [PATCH 14/14] fix --- .github/workflows/build-future.yml | 4 ++-- .github/workflows/build-latest.yml | 4 ++-- .github/workflows/build-testing.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index 870811fc..da6a3012 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -23,7 +23,7 @@ on: jobs: amd64: - uses: ./.github/workflows/build-image.yml + uses: ./.github/workflows/build.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -43,7 +43,7 @@ jobs: } arm64: - uses: ./.github/workflows/build-image.yml + uses: ./.github/workflows/build.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index e8995433..ee1ddaf7 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -24,7 +24,7 @@ on: jobs: amd64: - uses: ./.github/workflows/build-image.yml + uses: ./.github/workflows/build.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -46,7 +46,7 @@ jobs: } arm64: - uses: ./.github/workflows/build-image.yml + uses: ./.github/workflows/build.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index 4b50eaff..0010dc80 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -25,7 +25,7 @@ on: jobs: amd64: - uses: ./.github/workflows/build-image.yml + uses: ./.github/workflows/build.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -48,7 +48,7 @@ jobs: } arm64: - uses: ./.github/workflows/build-image.yml + uses: ./.github/workflows/build.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}