diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml index ff5d0fcf..f006d23c 100644 --- a/.github/workflows/build-future.yml +++ b/.github/workflows/build-future.yml @@ -4,37 +4,33 @@ name: Future # software that have been informally released to the futurenet network. on: - push: - branches: - - master - pull_request: - -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# master, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' + sha: + description: 'Sha to build' + type: 'string' + required: true jobs: - complete: - if: always() - needs: [manifest] - runs-on: ubuntu-latest - steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 - amd64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: amd64 - tag: future-amd64 + tag: ${{ inputs.tag-prefix }}future-amd64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -47,13 +43,14 @@ jobs: } arm64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: arm64 - tag: future-arm64 + tag: ${{ inputs.tag-prefix }}future-arm64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -74,5 +71,5 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag: future + tag: ${{ inputs.tag-prefix }}future images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 00000000..16b23e33 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,408 @@ +on: + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + sha: + description: 'Sha to build' + type: 'string' + required: true + arch: + description: 'Architecture to build the image for (amd64, arm64)' + type: 'string' + required: true + tag: + description: 'Tag to use on the image name' + type: 'string' + required: true + core_repo: + description: 'Git repo for stellar-core' + type: 'string' + default: 'https://github.com/stellar/stellar-core.git' + core_ref: + description: 'Git ref for the stellar-core repo' + type: 'string' + required: true + core_configure_flags: + description: 'CONFIGURE_FLAGS used when building stellar-core' + type: 'string' + default: '--disable-tests' + core_supports_enable_soroban_diagnostic_events: + description: 'Indicator whether stellar-core supports the ENABLE_SOROBAN_DIAGNOSTIC_EVENTS config' + type: 'string' + default: 'false' + core_build_runner_type: + description: 'The GitHub Runner instance type to build stellar-core on' + type: 'string' + default: 'ubuntu-latest' + go_ref: + description: 'Git ref for the stellar/go repo (stellar-horizon, stellar-friendbot)' + type: 'string' + required: true + xdr_ref: + description: 'Git ref for the stellar/rs-stellar-xdr repo' + type: 'string' + required: false + soroban_tools_ref: + description: 'Git ref for the stellar/soroban-tools repo (soroban-rpc)' + type: 'string' + required: true + soroban_rpc_build_runner_type: + description: 'The GitHub Runner instance type to build soroban-rpc on' + type: 'string' + default: 'ubuntu-latest' + test_matrix: + description: 'JSON matrix for the test job' + type: 'string' + required: true + outputs: + image: + description: 'Image pushed as a result of this build' + value: ${{ jobs.build.outputs.image }} + +env: + IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} + GO_REPO_BRANCH: ${{ inputs.go_ref }} + SOROBAN_TOOLS_REPO_BRANCH: ${{ inputs.soroban_tools_ref }} + CORE_REPO: ${{ inputs.core_repo }} + CORE_REPO_REF: ${{ inputs.core_ref }} + XDR_REPO_REF: ${{ inputs.xdr_ref }} + +jobs: + + build-stellar-core: + runs-on: ${{ inputs.core_build_runner_type }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Core Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f docker/Dockerfile.testing -t stellar-core:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + ${{ env.CORE_REPO }}#${{ env.CORE_REPO_REF }} + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' + - name: Upload Stellar-Core Image + uses: actions/upload-artifact@v2 + with: + name: image-stellar-core-${{ inputs.arch }} + path: /tmp/image + +# don't use caches on horizon builds as the git ref for it can be a branch name that refers to +# different commits over time, cache key won't invalidate correctly. + build-stellar-horizon: + runs-on: ubuntu-latest + steps: + - name: Checkout Quickstart for Horizon docker file + uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Setup buildx + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Horizon Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f Dockerfile.horizon --target builder + -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image + --build-arg REF="${{ env.GO_REPO_BRANCH }}" . + - name: Upload Stellar-Horizon Image + uses: actions/upload-artifact@v2 + with: + name: image-stellar-horizon-${{ inputs.arch }} + path: /tmp/image + +# don't use caches on friendbot builds as the git ref for it can be a branch name that refers to +# different commits over time, cache key won't invalidate correctly. + build-stellar-friendbot: + runs-on: ubuntu-latest + steps: + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Setup buildx + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - name: Build Stellar-Friendbot Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f services/friendbot/docker/Dockerfile -t stellar-friendbot:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + https://github.com/stellar/go.git#${{ env.GO_REPO_BRANCH }} + - name: Upload Stellar-Friendbot Image + uses: actions/upload-artifact@v2 + with: + name: image-stellar-friendbot-${{ inputs.arch }} + path: /tmp/image + + build-stellar-soroban-rpc: + runs-on: ${{ inputs.soroban_rpc_build_runner_type }} + steps: + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_TOOLS_REPO_BRANCH }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Soroban-Rpc Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f cmd/soroban-rpc/docker/Dockerfile --target build + -t stellar-soroban-rpc:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true + https://github.com/stellar/soroban-tools.git#${{ env.SOROBAN_TOOLS_REPO_BRANCH }} + - name: Upload Stellar-Soroban-Rpc Image + uses: actions/upload-artifact@v2 + with: + name: image-stellar-soroban-rpc-${{ inputs.arch }} + path: /tmp/image + + build-rs-stellar-xdr: + runs-on: ubuntu-latest + steps: + - name: Checkout Quickstart for Horizon docker file + uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - id: cache + uses: actions/cache@v3 + with: + path: /tmp/image + key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} + - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - if: steps.cache.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f + - if: steps.cache.outputs.cache-hit != 'true' + name: Build Stellar-Rs-Xdr Image + run: > + docker buildx build --platform linux/${{ inputs.arch }} + -f Dockerfile.xdr --target builder + -t stellar-rs-xdr:${{ inputs.arch }} + -o type=docker,dest=/tmp/image + --build-arg REPO=https://github.com/stellar/rs-stellar-xdr.git + --build-arg REF="${{ env.XDR_REPO_REF }}" . + - name: Upload Stellar-Rs-Xdr Image + uses: actions/upload-artifact@v2 + with: + name: image-rs-stellar-xdr-${{ inputs.arch }} + path: /tmp/image + + build: + needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] + outputs: + image: ${{ steps.image.outputs.name }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + - name: Download Stellar XDR + uses: actions/download-artifact@v2 + with: + name: image-rs-stellar-xdr-${{ inputs.arch }} + path: /tmp/stellar-xdr + - name: Download Stellar-Core Image + uses: actions/download-artifact@v2 + with: + name: image-stellar-core-${{ inputs.arch }} + path: /tmp/stellar-core + - name: Download Stellar-Horizon Image + uses: actions/download-artifact@v2 + with: + name: image-stellar-horizon-${{ inputs.arch }} + path: /tmp/stellar-horizon + - name: Download Stellar-Friendbot Image + uses: actions/download-artifact@v2 + with: + name: image-stellar-friendbot-${{ inputs.arch }} + path: /tmp/stellar-friendbot + - name: Download Stellar-Soroban-Rpc Image + uses: actions/download-artifact@v2 + with: + name: image-stellar-soroban-rpc-${{ inputs.arch }} + path: /tmp/stellar-soroban-rpc + - name: Load Stellar-Core Image + run: docker load -i /tmp/stellar-core/image + - name: Load Stellar-Horizon Image + run: docker load -i /tmp/stellar-horizon/image + - name: Load Stellar-Friendbot Image + run: docker load -i /tmp/stellar-friendbot/image + - name: Load Stellar-Soroban-Rpc Image + run: docker load -i /tmp/stellar-soroban-rpc/image + - name: Load Stellar-Rs-Xdr Image + run: docker load -i /tmp/stellar-xdr/image + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - id: image + name: Image Name + run: echo "name=$IMAGE" >> $GITHUB_OUTPUT + - name: Pull Base Image + run: docker pull --platform linux/${{ inputs.arch }} ubuntu:22.04 + # Docker buildx cannot be used to build the dev quickstart image because + # buildx does not yet support importing existing images, like the core and + # horizon images above, into a buildx builder's cache. Buildx would be + # preferred because it can output a smaller image file faster than docker + # save can. Once buildx supports it we can update. + # https://github.com/docker/buildx/issues/847 + - name: Build Quickstart Image + run: > + docker build + --platform linux/${{ inputs.arch }} + -f Dockerfile + -t $IMAGE + --label org.opencontainers.image.revision="${{ inputs.sha }}" + . + --build-arg REVISION="${{ inputs.sha }}" + --build-arg STELLAR_XDR_IMAGE_REF=stellar-rs-xdr:${{ inputs.arch }} + --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:${{ inputs.arch }} + --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ inputs.arch }} + --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ inputs.arch }} + --build-arg SOROBAN_RPC_IMAGE_REF=stellar-soroban-rpc:${{ inputs.arch }} + --build-arg CORE_SUPPORTS_ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=${{ inputs.core_supports_enable_soroban_diagnostic_events }} + - name: Save Quickstart Image + run: docker save $IMAGE -o /tmp/image + - name: Upload Quickstart Image + uses: actions/upload-artifact@v2 + with: + name: image-${{ inputs.arch }} + path: /tmp/image + + test: + needs: build + strategy: + matrix: ${{ fromJSON(inputs.test_matrix) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - name: Download Quickstart Image + uses: actions/download-artifact@v2 + with: + name: image-${{ inputs.arch }} + path: /tmp/ + - name: Load Quickstart Image + run: docker load -i /tmp/image + - if: inputs.arch == 'arm64' + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + with: + platforms: arm64 + - name: Run Quickstart Image + run: docker run --platform linux/${{ inputs.arch }} --rm -d -p "8000:8000" -p "11626:11626" --name stellar $IMAGE --${{ matrix.network }} ${{ matrix.options }} + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ^1 + - name: Sleep until supervisor is up + run: sleep 10 + - name: Run core test + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & + go run test_core.go + curl http://localhost:11626/info + - name: Run horizon up test + run: | + docker logs stellar -f & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_horizon_up.go + curl http://localhost:8000 + - name: Run horizon ingesting test + if: ${{ matrix.network != 'pubnet' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_horizon_ingesting.go + curl http://localhost:8000 + - name: Run friendbot test + if: ${{ matrix.network == 'local' }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & + echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & + go run test_friendbot.go + - name: Run soroban rpc test + if: ${{ contains(matrix.options, '--enable-soroban-rpc') }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f soroban-rpc" | docker exec -i stellar sh & + go run test_soroban_rpc_up.go + + push-pr: + # Push image to registry after build for pull requests from a local branch. + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + needs: build + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - id: push + uses: ./.github/actions/push + with: + head_sha: ${{ inputs.sha }} + artifact_name: image-${{ inputs.arch }} + artifact_image_file: image + arch: ${{ inputs.arch }} + image: ${{ env.IMAGE }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + + push-release: + # Push image to registry after test for master. + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + needs: [build, test] + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.sha }} + - id: push + uses: ./.github/actions/push + with: + head_sha: ${{ inputs.sha }} + artifact_name: image-${{ inputs.arch }} + artifact_image_file: image + arch: ${{ inputs.arch }} + image: ${{ env.IMAGE }} + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml index c9fa51b3..a66c1916 100644 --- a/.github/workflows/build-latest.yml +++ b/.github/workflows/build-latest.yml @@ -5,37 +5,33 @@ name: Latest # tag for software that's been released which is not a release candidate. on: - push: - branches: - - master - pull_request: - -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# master, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' + sha: + description: 'Sha to build' + type: 'string' + required: true jobs: - complete: - if: always() - needs: [manifest] - runs-on: ubuntu-latest - steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 - amd64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: amd64 - tag: latest-amd64 + tag: ${{ inputs.tag-prefix }}latest-amd64 xdr_ref: v20.0.2 core_ref: v20.0.2 go_ref: horizon-v2.27.0 @@ -50,13 +46,14 @@ jobs: } arm64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: arm64 - tag: latest-arm64 + tag: ${{ inputs.tag-prefix }}latest-arm64 xdr_ref: v20.0.2 core_ref: v20.0.2 core_build_runner_type: ubuntu-latest-16-cores @@ -79,5 +76,5 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag: latest + tag: ${{ inputs.tag-prefix }}latest images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml index 065f75d9..3815c17c 100644 --- a/.github/workflows/build-testing.yml +++ b/.github/workflows/build-testing.yml @@ -6,37 +6,33 @@ name: Testing # release. on: - push: - branches: - - master - pull_request: - -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# master, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + inputs: + tag-prefix: + description: 'Prefix for the tag name' + type: 'string' + default: '' + sha: + description: 'Sha to build' + type: 'string' + required: true jobs: - complete: - if: always() - needs: [manifest] - runs-on: ubuntu-latest - steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 - amd64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: amd64 - tag: testing-amd64 + tag: ${{ inputs.tag-prefix }}testing-amd64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -52,13 +48,14 @@ jobs: } arm64: - uses: ./.github/workflows/build.yml + uses: ./.github/workflows/build-image.yml secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: + sha: ${{ inputs.sha }} arch: arm64 - tag: testing-arm64 + tag: ${{ inputs.tag-prefix }}testing-arm64 xdr_ref: v20.0.2 core_ref: v20.1.0 core_supports_enable_soroban_diagnostic_events: "true" @@ -82,5 +79,5 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} with: - tag: testing + tag: ${{ inputs.tag-prefix }}testing images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4a9f144..ccdf7587 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,422 +1,69 @@ +name: Build + on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - arch: - description: 'Architecture to build the image for (amd64, arm64)' - type: 'string' - required: true - tag: - description: 'Tag to use on the image name' - type: 'string' - required: true - core_repo: - description: 'Git repo for stellar-core' - type: 'string' - default: 'https://github.com/stellar/stellar-core.git' - core_ref: - description: 'Git ref for the stellar-core repo' - type: 'string' - required: true - core_configure_flags: - description: 'CONFIGURE_FLAGS used when building stellar-core' - type: 'string' - default: '--disable-tests' - core_supports_enable_soroban_diagnostic_events: - description: 'Indicator whether stellar-core supports the ENABLE_SOROBAN_DIAGNOSTIC_EVENTS config' - type: 'string' - default: 'false' - core_build_runner_type: - description: 'The GitHub Runner instance type to build stellar-core on' - type: 'string' - default: 'ubuntu-latest' - go_ref: - description: 'Git ref for the stellar/go repo (stellar-horizon, stellar-friendbot)' - type: 'string' - required: true - xdr_ref: - description: 'Git ref for the stellar/rs-stellar-xdr repo' - type: 'string' - required: false - soroban_tools_ref: - description: 'Git ref for the stellar/soroban-tools repo (soroban-rpc)' - type: 'string' - required: true - soroban_rpc_build_runner_type: - description: 'The GitHub Runner instance type to build soroban-rpc on' - type: 'string' - default: 'ubuntu-latest' - test_matrix: - description: 'JSON matrix for the test job' - type: 'string' - required: true - outputs: - image: - description: 'Image pushed as a result of this build' - value: ${{ jobs.build.outputs.image }} + push: + branches: + - master + pull_request: -env: - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, github.event_name == 'pull_request' && format('pr-{0}-{1}', github.event.pull_request.number, inputs.tag) || inputs.tag) }} - GO_REPO_BRANCH: ${{ inputs.go_ref }} - SOROBAN_TOOLS_REPO_BRANCH: ${{ inputs.soroban_tools_ref }} - CORE_REPO: ${{ inputs.core_repo }} - CORE_REPO_REF: ${{ inputs.core_ref }} - XDR_REPO_REF: ${{ inputs.xdr_ref }} +# Prevent more than one build of this workflow for a branch to be running at the +# same time, and if multiple are queued, only run the latest, cancelling any +# already running build. The exception being any protected branch, such as +# master, where a build for every commit will run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true jobs: - build-stellar-core: - runs-on: ${{ inputs.core_build_runner_type }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-stellar-core-${{ inputs.arch }}-${{ env.CORE_REPO_REF }}-${{ inputs.core_configure_flags }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Core Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f docker/Dockerfile.testing -t stellar-core:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - ${{ env.CORE_REPO }}#${{ env.CORE_REPO_REF }} - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' - - name: Upload Stellar-Core Image - uses: actions/upload-artifact@v2 - with: - name: image-stellar-core-${{ inputs.arch }} - path: /tmp/image - -# don't use caches on horizon builds as the git ref for it can be a branch name that refers to -# different commits over time, cache key won't invalidate correctly. - build-stellar-horizon: + complete: + if: always() + needs: [latest, testing, future] runs-on: ubuntu-latest steps: - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ env.HEAD_SHA }} - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Horizon Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.horizon --target builder - -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image - --build-arg REF="${{ env.GO_REPO_BRANCH }}" . - - name: Upload Stellar-Horizon Image - uses: actions/upload-artifact@v2 - with: - name: image-stellar-horizon-${{ inputs.arch }} - path: /tmp/image - -# don't use caches on friendbot builds as the git ref for it can be a branch name that refers to -# different commits over time, cache key won't invalidate correctly. - build-stellar-friendbot: - runs-on: ubuntu-latest - steps: - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Friendbot Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f services/friendbot/docker/Dockerfile -t stellar-friendbot:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - https://github.com/stellar/go.git#${{ env.GO_REPO_BRANCH }} - - name: Upload Stellar-Friendbot Image - uses: actions/upload-artifact@v2 - with: - name: image-stellar-friendbot-${{ inputs.arch }} - path: /tmp/image - - build-stellar-soroban-rpc: - runs-on: ${{ inputs.soroban_rpc_build_runner_type }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-stellar-soroban-rpc-${{ inputs.arch }}-${{ env.SOROBAN_TOOLS_REPO_BRANCH }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Soroban-Rpc Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f cmd/soroban-rpc/docker/Dockerfile --target build - -t stellar-soroban-rpc:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=true - https://github.com/stellar/soroban-tools.git#${{ env.SOROBAN_TOOLS_REPO_BRANCH }} - - name: Upload Stellar-Soroban-Rpc Image - uses: actions/upload-artifact@v2 - with: - name: image-stellar-soroban-rpc-${{ inputs.arch }} - path: /tmp/image + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 - build-rs-stellar-xdr: + tag-prefix: + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} runs-on: ubuntu-latest - steps: - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ env.HEAD_SHA }} - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-rs-stellar-xdr-${{ inputs.arch }}-${{ env.XDR_REPO_REF }} - - if: steps.cache.outputs.cache-hit != 'true' && inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - if: steps.cache.outputs.cache-hit != 'true' - name: Build Stellar-Rs-Xdr Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.xdr --target builder - -t stellar-rs-xdr:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REPO=https://github.com/stellar/rs-stellar-xdr.git - --build-arg REF="${{ env.XDR_REPO_REF }}" . - - name: Upload Stellar-Rs-Xdr Image - uses: actions/upload-artifact@v2 - with: - name: image-rs-stellar-xdr-${{ inputs.arch }} - path: /tmp/image - - build: - needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-soroban-rpc] outputs: - image: ${{ steps.image.outputs.name }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ env.HEAD_SHA }} - - name: Download Stellar XDR - uses: actions/download-artifact@v2 - with: - name: image-rs-stellar-xdr-${{ inputs.arch }} - path: /tmp/stellar-xdr - - name: Download Stellar-Core Image - uses: actions/download-artifact@v2 - with: - name: image-stellar-core-${{ inputs.arch }} - path: /tmp/stellar-core - - name: Download Stellar-Horizon Image - uses: actions/download-artifact@v2 - with: - name: image-stellar-horizon-${{ inputs.arch }} - path: /tmp/stellar-horizon - - name: Download Stellar-Friendbot Image - uses: actions/download-artifact@v2 - with: - name: image-stellar-friendbot-${{ inputs.arch }} - path: /tmp/stellar-friendbot - - name: Download Stellar-Soroban-Rpc Image - uses: actions/download-artifact@v2 - with: - name: image-stellar-soroban-rpc-${{ inputs.arch }} - path: /tmp/stellar-soroban-rpc - - name: Load Stellar-Core Image - run: docker load -i /tmp/stellar-core/image - - name: Load Stellar-Horizon Image - run: docker load -i /tmp/stellar-horizon/image - - name: Load Stellar-Friendbot Image - run: docker load -i /tmp/stellar-friendbot/image - - name: Load Stellar-Soroban-Rpc Image - run: docker load -i /tmp/stellar-soroban-rpc/image - - name: Load Stellar-Rs-Xdr Image - run: docker load -i /tmp/stellar-xdr/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - id: image - name: Image Name - run: echo "name=$IMAGE" >> $GITHUB_OUTPUT - - name: Pull Base Image - run: docker pull --platform linux/${{ inputs.arch }} ubuntu:22.04 - # Docker buildx cannot be used to build the dev quickstart image because - # buildx does not yet support importing existing images, like the core and - # horizon images above, into a buildx builder's cache. Buildx would be - # preferred because it can output a smaller image file faster than docker - # save can. Once buildx supports it we can update. - # https://github.com/docker/buildx/issues/847 - - name: Build Quickstart Image - run: > - docker build - --platform linux/${{ inputs.arch }} - -f Dockerfile - -t $IMAGE - --label org.opencontainers.image.revision="${{ env.HEAD_SHA }}" - . - --build-arg REVISION="${{ env.HEAD_SHA }}" - --build-arg STELLAR_XDR_IMAGE_REF=stellar-rs-xdr:${{ inputs.arch }} - --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:${{ inputs.arch }} - --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ inputs.arch }} - --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ inputs.arch }} - --build-arg SOROBAN_RPC_IMAGE_REF=stellar-soroban-rpc:${{ inputs.arch }} - --build-arg CORE_SUPPORTS_ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=${{ inputs.core_supports_enable_soroban_diagnostic_events }} - - name: Save Quickstart Image - run: docker save $IMAGE -o /tmp/image - - name: Upload Quickstart Image - uses: actions/upload-artifact@v2 - with: - name: image-${{ inputs.arch }} - path: /tmp/image - - test: - needs: build - strategy: - matrix: ${{ fromJSON(inputs.test_matrix) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ env.HEAD_SHA }} - - name: Download Quickstart Image - uses: actions/download-artifact@v2 - with: - name: image-${{ inputs.arch }} - path: /tmp/ - - name: Load Quickstart Image - run: docker load -i /tmp/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - - name: Run Quickstart Image - run: docker run --platform linux/${{ inputs.arch }} --rm -d -p "8000:8000" -p "11626:11626" --name stellar $IMAGE --${{ matrix.network }} ${{ matrix.options }} - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ^1 - - name: Sleep until supervisor is up - run: sleep 10 - - name: Run core test - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & - go run test_core.go - curl http://localhost:11626/info - - name: Run horizon up test - run: | - docker logs stellar -f & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_horizon_up.go - curl http://localhost:8000 - - name: Run horizon ingesting test - if: ${{ matrix.network != 'pubnet' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_horizon_ingesting.go - curl http://localhost:8000 - - name: Run friendbot test - if: ${{ matrix.network == 'local' }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & - echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & - go run test_friendbot.go - - name: Run soroban rpc test - if: ${{ contains(matrix.options, '--enable-soroban-rpc') }} - run: | - docker logs stellar -f & - echo "supervisorctl tail -f soroban-rpc" | docker exec -i stellar sh & - go run test_soroban_rpc_up.go - - push-pr: - # Push image to registry after build for pull requests from a local branch. - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} - needs: build - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest + tag-prefix: ${{ steps.release.outputs.tag-prefix }} steps: - uses: actions/checkout@v2 with: - ref: ${{ env.HEAD_SHA }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ env.HEAD_SHA }} - artifact_name: image-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - calc-push-release-version: - # Calculate the version of the release. - # TODO: Uncomment: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - runs-on: ubuntu-latest - outputs: - count: ${{ steps.count.outputs.count }} - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ env.HEAD_SHA }} - - id: count + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - id: release run: count="$(git rev-list HEAD --count --first-parent)" - echo "count=$count" >> $GITHUB_OUTPUT + echo "tag-prefix=v${count}-" >> $GITHUB_OUTPUT - push-release: - # Push image to registry after test for master. - # TODO: Uncomment: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - needs: [build, test, calc-push-release-version] - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ env.HEAD_SHA }} - - run: - echo v${{ needs.calc-push-release-version.outputs.count }}-${{ env.IMAGE }} - # TODO: Uncomment: - id: push - # TODO: Uncomment: uses: ./.github/actions/push - # TODO: Uncomment: with: - # TODO: Uncomment: head_sha: ${{ env.HEAD_SHA }} - # TODO: Uncomment: artifact_name: image-${{ inputs.arch }} - # TODO: Uncomment: artifact_image_file: image - # TODO: Uncomment: arch: ${{ inputs.arch }} - # TODO: Uncomment: image: v${{ needs.calc-push-release-version.outputs.count }}-${{ env.IMAGE }} - # TODO: Uncomment: registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - # TODO: Uncomment: username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - # TODO: Uncomment: password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + latest: + needs: [tag-prefix] + uses: ./.github/workflows/build-latest.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} + + testing: + needs: [tag-prefix] + uses: ./.github/workflows/build-testing.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} + + future: + needs: [tag-prefix] + uses: ./.github/workflows/build-future.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + with: + tag-prefix: ${{ needs.tag-prefix.outputs.tag-prefix }} + sha: ${{ github.event.pull_request.head.sha || github.sha }}