From e2a884152293040ab1fb6d5312232b64286afa64 Mon Sep 17 00:00:00 2001 From: frank zhu Date: Wed, 14 Aug 2024 23:55:32 -0500 Subject: [PATCH] refactor: use only goreleaser to build unsigned chainlink images in one workflow (#14034) * refactor: use only goreleaser to build unsigned chainlink images in one workflow * fix: use branches-ignore filter instead * rename and switch runner * temp switch back name because of env protection rule * back to use the refactored name * update goreleaser yaml * add new docker inputs, rename IMAGE_NAME, add goreleaser build-sign-publish workflow * add output image name and digest to github summary * refactor gha workflow names and add output image name and digest to build-develop * remove unnecessary outputs * add a git_ref validation job * temp delete workflows for easier testing * add if conditional to validate step * fix metric name and add debug log * update docker registry * no goreleaser output since we don't use the goreleaser/action * remove debug log and use bash shell * fix formatting * remove root images from goreleaser yaml * use custom setup-go * fix typo * use tee instead * add back setup-go and refactor output summary step * update with new filename and workflow trigger * fix docker registry input * remove role-duration input * change conditional * revert temp gha workflow delete commit * sync with origin develop * refactor trigger based on push and pr label * fix install remote plugins bug include * add new docker builds for plugins and update dockerfile * add goreleaser --split to gha and refactor action_utils script * fix add shell * fix metrics job name and publish docker manifest files * fix image_templates goreleaser * fix check artifacts.json and metrics name * fix if not end * ls -al dist * add --single-target flag and split checksum * remove split in checksum * remove --single-target and update output artifact.json path * cat artifacts.json * use ubuntu-latest runner * update build-publish workflow output summary step * build on every pr - conditional publish | add workflow_dispatch trigger * add workflow_dispatch conditional build-publish * fix typo * fix typo * use ubuntu-20.04 runner * fix conditional * add comment --- .../goreleaser-build-sign-publish/action.yml | 41 ++++-- .../action_utils | 32 +++-- .../workflows/build-publish-develop-pr.yml | 119 ++++++++++++++++++ .github/workflows/build-publish-develop.yml | 69 ---------- .github/workflows/build-publish-pr.yml | 66 ---------- .github/workflows/build-publish.yml | 65 +++++++++- .github/workflows/build.yml | 51 -------- .../goreleaser-build-publish-develop.yml | 52 -------- .goreleaser.develop.yaml | 65 ++++++---- core/chainlink.goreleaser.Dockerfile | 5 +- tools/bin/goreleaser_utils | 8 +- 11 files changed, 283 insertions(+), 290 deletions(-) create mode 100644 .github/workflows/build-publish-develop-pr.yml delete mode 100644 .github/workflows/build-publish-develop.yml delete mode 100644 .github/workflows/build-publish-pr.yml delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/goreleaser-build-publish-develop.yml diff --git a/.github/actions/goreleaser-build-sign-publish/action.yml b/.github/actions/goreleaser-build-sign-publish/action.yml index f4b2111bea5..c279c2f9290 100644 --- a/.github/actions/goreleaser-build-sign-publish/action.yml +++ b/.github/actions/goreleaser-build-sign-publish/action.yml @@ -29,10 +29,13 @@ inputs: description: The docker registry default: localhost:5001 required: false - # snapshot inputs - enable-goreleaser-snapshot: - description: Enable goreleaser build / release snapshot - default: "false" + docker-image-name: + description: The docker image name + default: chainlink + required: false + docker-image-tag: + description: The docker image tag + default: develop required: false # goreleaser inputs goreleaser-exec: @@ -43,6 +46,17 @@ inputs: description: "The goreleaser configuration yaml" default: ".goreleaser.yaml" required: false + enable-goreleaser-snapshot: + description: Enable goreleaser build / release snapshot + default: "false" + required: false + enable-goreleaser-split: + description: Enable goreleaser split and merge builds + default: "false" + required: false + goreleaser-split-arch: + description: The architecture to split the goreleaser build + required: false # signing inputs enable-cosign: description: Enable signing of docker images @@ -57,13 +71,6 @@ inputs: cosign-password: description: The password to decrypt the cosign private key needed to sign the image required: false -outputs: - goreleaser-metadata: - description: "Build result metadata" - value: ${{ steps.goreleaser.outputs.metadata }} - goreleaser-artifacts: - description: "Build result artifacts" - value: ${{ steps.goreleaser.outputs.artifacts }} runs: using: composite steps: @@ -97,14 +104,22 @@ runs: uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 with: registry: ${{ inputs.docker-registry }} - - name: Goreleaser release - id: goreleaser + - name: Set goreleaser split env + if: inputs.enable-goreleaser-split == 'true' + shell: bash + run: | + echo "GOOS=linux" | tee -a $GITHUB_ENV + echo "GOARCH=${{ inputs.goreleaser-split-arch }}" | tee -a $GITHUB_ENV + - name: Run goreleaser release shell: bash env: ENABLE_COSIGN: ${{ inputs.enable-cosign }} ENABLE_GORELEASER_SNAPSHOT: ${{ inputs.enable-goreleaser-snapshot }} + ENABLE_GORELEASER_SPLIT: ${{ inputs.enable-goreleaser-split }} ENABLE_DOCKER_PUBLISH: ${{ inputs.enable-docker-publish }} IMAGE_PREFIX: ${{ inputs.docker-registry }} + IMAGE_NAME: ${{ inputs.docker-image-name }} + IMAGE_TAG: ${{ inputs.docker-image-tag }} GORELEASER_EXEC: ${{ inputs.goreleaser-exec }} GORELEASER_CONFIG: ${{ inputs.goreleaser-config }} COSIGN_PASSWORD: ${{ inputs.cosign-password }} diff --git a/.github/actions/goreleaser-build-sign-publish/action_utils b/.github/actions/goreleaser-build-sign-publish/action_utils index 4aac78d6fcc..051e0763fbd 100755 --- a/.github/actions/goreleaser-build-sign-publish/action_utils +++ b/.github/actions/goreleaser-build-sign-publish/action_utils @@ -4,6 +4,7 @@ set -euo pipefail ENABLE_COSIGN=${ENABLE_COSIGN:-false} ENABLE_GORELEASER_SNAPSHOT=${ENABLE_GORELEASER_SNAPSHOT:-false} +ENABLE_GORELEASER_SPLIT=${ENABLE_GORELEASER_SPLIT:-false} ENABLE_DOCKER_PUBLISH=${ENABLE_DOCKER_PUBLISH:-false} COSIGN_PASSWORD=${COSIGN_PASSWORD:-""} GORELEASER_EXEC=${GORELEASER_EXEC:-goreleaser} @@ -27,8 +28,12 @@ _publish_snapshot_manifests() { local docker_manifest_extra_args=$DOCKER_MANIFEST_EXTRA_ARGS local full_sha=$(git rev-parse HEAD) local images=$(docker images --filter "label=org.opencontainers.image.revision=$full_sha" --format "{{.Repository}}:{{.Tag}}" | sort) - local arches=(amd64 arm64) local raw_manifest_lists="" + if [[ $ENABLE_GORELEASER_SPLIT == "true" ]]; then + local arches=(${GOARCH:-""}) + else + local arches=(amd64 arm64) + fi for image in $images; do for arch in "${arches[@]}"; do image=${image%"-$arch"} @@ -51,22 +56,35 @@ _publish_snapshot_manifests() { # wrapper function to invoke goreleaser release goreleaser_release() { + goreleaser_flags=() + + # set goreleaser flags + if [[ $ENABLE_GORELEASER_SNAPSHOT == "true" ]]; then + goreleaser_flags+=("--snapshot") + goreleaser_flags+=("--clean") + fi + if [[ $ENABLE_GORELEASER_SPLIT == "true" ]]; then + goreleaser_flags+=("--split") + fi + flags=$(printf "%s " "${goreleaser_flags[@]}") + flags=$(echo "$flags" | sed 's/ *$//') + if [[ $ENABLE_COSIGN == "true" ]]; then echo "$COSIGN_PUBLIC_KEY" > cosign.pub echo "$COSIGN_PRIVATE_KEY" > cosign.key fi + if [[ -n $MACOS_SDK_DIR ]]; then MACOS_SDK_DIR=$(echo "$(cd "$(dirname "$MACOS_SDK_DIR")" || exit; pwd)/$(basename "$MACOS_SDK_DIR")") fi - if [[ $ENABLE_GORELEASER_SNAPSHOT == "true" ]]; then - $GORELEASER_EXEC release --snapshot --clean --config "$GORELEASER_CONFIG" "$@" - if [[ $ENABLE_DOCKER_PUBLISH == "true" ]]; then + + $GORELEASER_EXEC release ${flags} --config "$GORELEASER_CONFIG" "$@" + + if [[ $ENABLE_DOCKER_PUBLISH == "true" ]]; then _publish_snapshot_images _publish_snapshot_manifests - fi - else - $GORELEASER_EXEC release --clean --config "$GORELEASER_CONFIG" "$@" fi + if [[ $ENABLE_COSIGN == "true" ]]; then rm -rf cosign.pub rm -rf cosign.key diff --git a/.github/workflows/build-publish-develop-pr.yml b/.github/workflows/build-publish-develop-pr.yml new file mode 100644 index 00000000000..c0c3a17824b --- /dev/null +++ b/.github/workflows/build-publish-develop-pr.yml @@ -0,0 +1,119 @@ +name: "Build and Publish Chainlink" + +on: + pull_request: + push: + branches: + - develop + - "release/**" + workflow_dispatch: + inputs: + git_ref: + description: "The git ref to check out" + required: true + build-publish: + description: "Whether to build and publish - defaults to just build" + required: false + default: "false" + +env: + GIT_REF: ${{ github.event.inputs.git_ref || github.ref }} + +jobs: + goreleaser-build-publish-chainlink: + runs-on: ubuntu-20.04 + permissions: + id-token: write + contents: read + strategy: + matrix: + goarch: [amd64, arm64] + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + ref: ${{ env.GIT_REF }} + + # This gets the image tag and whether to publish the image based on the event type + # PR builds: pr-- (if label 'build-publish' is present publishes the image) + # develop builds: develop- + # release builds: release- + # manual builds: (if build-publish is true publishes the image) + - name: Get image tag + id: get-image-tag + run: | + short_sha=$(git rev-parse --short HEAD) + echo "build-publish=false" | tee -a $GITHUB_OUTPUT + if [[ ${{ github.event_name }} == 'push' ]]; then + if [[ ${{ github.ref_name }} == 'release/'* ]]; then + echo "image-tag=release-${short_sha}" | tee -a $GITHUB_OUTPUT + echo "build-publish=true" | tee -a $GITHUB_OUTPUT + else + echo "image-tag=develop-${short_sha}" | tee -a $GITHUB_OUTPUT + echo "build-publish=true" | tee -a $GITHUB_OUTPUT + fi + elif [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then + echo "image-tag=${short_sha}" | tee -a $GITHUB_OUTPUT + echo "build-publish=${{ github.event.inputs.build-publish }}" | tee -a $GITHUB_OUTPUT + else + if [[ ${{ github.event_name }} == "pull_request" ]]; then + echo "image-tag=pr-${{ github.event.number }}-${short_sha}" | tee -a $GITHUB_OUTPUT + if [[ ${{ contains(github.event.pull_request.labels.*.name, 'build-publish') }} == "true" ]]; then + echo "build-publish=true" | tee -a $GITHUB_OUTPUT + fi + fi + fi + + - name: Configure aws credentials + if: steps.get-image-tag.outputs.build-publish == 'true' + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_BUILD_PUBLISH_DEVELOP_PR }} + aws-region: ${{ secrets.AWS_REGION }} + mask-aws-account-id: true + role-session-name: goreleaser-build-publish-chainlink + + - name: Build and publish images + uses: ./.github/actions/goreleaser-build-sign-publish + with: + enable-docker-publish: ${{ steps.get-image-tag.outputs.build-publish }} + docker-registry: ${{ secrets.AWS_SDLC_ECR_HOSTNAME }} + docker-image-name: chainlink + docker-image-tag: ${{ steps.get-image-tag.outputs.image-tag }} + enable-goreleaser-snapshot: "true" + enable-goreleaser-split: "true" + goreleaser-split-arch: ${{ matrix.goarch }} + goreleaser-exec: ./tools/bin/goreleaser_wrapper + goreleaser-config: .goreleaser.develop.yaml + goreleaser-key: ${{ secrets.GORELEASER_KEY }} + zig-version: 0.11.0 + + - name: Output image name and digest + if: steps.get-image-tag.outputs.build-publish == 'true' + shell: bash + run: | + # need to check if artifacts.json exists because goreleaser splits the build + if [[ -f dist/artifacts.json ]]; then + artifact_path="dist/artifacts.json" + else + artifact_path="dist/linux_${{ matrix.goarch }}/artifacts.json" + cat dist/linux_${{ matrix.goarch }}/artifacts.json + fi + echo "### Docker Images" | tee -a "$GITHUB_STEP_SUMMARY" + jq -r '.[] | select(.type == "Docker Image") | "`\(.goarch)-image`: \(.name)"' ${artifact_path} >> output.txt + jq -r '.[] | select(.type == "Archive") | "`\(.goarch)-digest`: \(.extra.Checksum)"' ${artifact_path} >> output.txt + while read -r line; do + echo "$line" | tee -a "$GITHUB_STEP_SUMMARY" + done < output.txt + + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 + with: + id: goreleaser-build-publish + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: goreleaser-build-publish-chainlink (${{ matrix.goarch }}) + continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/build-publish-develop.yml b/.github/workflows/build-publish-develop.yml deleted file mode 100644 index 8c6e5e76ac4..00000000000 --- a/.github/workflows/build-publish-develop.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: "Push develop to private ECR" - -on: - push: - branches: - - develop - workflow_dispatch: - inputs: - git_ref: - description: "Git ref (commit SHA, branch name, tag name, etc.) to checkout" - required: true -env: - GIT_REF: ${{ github.event.inputs.git_ref || github.ref }} - -jobs: - push-chainlink-develop: - runs-on: ubuntu-20.04 - environment: build-develop - permissions: - id-token: write - contents: read - strategy: - matrix: - image: - - name: "" - dockerfile: core/chainlink.Dockerfile - tag-suffix: "" - - name: (plugins) - dockerfile: plugins/chainlink.Dockerfile - tag-suffix: -plugins - name: push-chainlink-develop ${{ matrix.image.name }} - steps: - - name: Checkout repository - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - ref: ${{ env.GIT_REF }} - # When this is ran from manual workflow_dispatch, the github.sha may be - # different than the checked out commit sha. The core build uses this - # commit sha as build metadata, so we need to make sure it's correct. - - name: Get checked out git ref - if: github.event.inputs.git_ref - id: git-ref - run: echo "checked-out=$(git rev-parse HEAD)" | tee -a "${GITHUB_OUTPUT}" - - name: Build, sign and publish chainlink image - uses: ./.github/actions/build-sign-publish-chainlink - with: - publish: true - aws-role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_ARN }} - aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }} - aws-region: ${{ secrets.AWS_REGION }} - ecr-hostname: ${{ secrets.AWS_DEVELOP_ECR_HOSTNAME }} - ecr-image-name: chainlink - ecr-tag-suffix: ${{ matrix.image.tag-suffix }} - dockerfile: ${{ matrix.image.dockerfile }} - dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} - dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} - git-commit-sha: ${{ steps.git-ref.outputs.checked-out || github.sha }} - - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: build-chainlink-develop - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: push-chainlink-develop ${{ matrix.image.name }} - continue-on-error: true diff --git a/.github/workflows/build-publish-pr.yml b/.github/workflows/build-publish-pr.yml deleted file mode 100644 index 36eac61cab0..00000000000 --- a/.github/workflows/build-publish-pr.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: "Build and Publish from PR" - -## -# This workflow builds and publishes a Docker image for Chainlink from a PR. -# It has its own special IAM role, does not sign the image, and publishes to -# a special ECR repo. -## - -on: - pull_request: - -jobs: - build-publish-untrusted: - if: ${{ ! startsWith(github.ref_name, 'release/') || (! startsWith(github.head_ref, 'release/') && ! startsWith(github.ref_name, 'chore/'))}} - runs-on: ubuntu-20.04 - environment: sdlc - permissions: - id-token: write - contents: read - env: - ECR_IMAGE_NAME: crib-chainlink-untrusted - steps: - - name: Checkout repository - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - - name: Git Short SHA - shell: bash - env: - GIT_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - echo "GIT_SHORT_SHA=${GIT_PR_HEAD_SHA:0:7}" | tee -a "$GITHUB_ENV" - - - name: Check if image exists - id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - with: - repository: ${{ env.ECR_IMAGE_NAME}} - tag: sha-${{ env.GIT_SHORT_SHA }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_OIDC_IAM_ROLE_PUBLISH_PR_ARN }} - - - name: Build and publish chainlink image - if: steps.check-image.outputs.exists == 'false' - uses: ./.github/actions/build-sign-publish-chainlink - with: - publish: true - aws-role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_PUBLISH_PR_ARN }} - aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS_DEFAULT }} - aws-region: ${{ secrets.AWS_REGION }} - sign-images: false - ecr-hostname: ${{ secrets.AWS_SDLC_ECR_HOSTNAME }} - ecr-image-name: ${{ env.ECR_IMAGE_NAME }} - dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} - dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} - - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: build-chainlink-pr - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: build-publish-untrusted - continue-on-error: true diff --git a/.github/workflows/build-publish.yml b/.github/workflows/build-publish.yml index 1a3c6546a6e..0941455a169 100644 --- a/.github/workflows/build-publish.yml +++ b/.github/workflows/build-publish.yml @@ -1,4 +1,4 @@ -name: "Build Chainlink and Publish" +name: "Build, Sign and Publish Chainlink" on: # Mimics old circleci behaviour @@ -59,6 +59,7 @@ jobs: dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} verify-signature: true + - name: Collect Metrics if: always() id: collect-gha-metrics @@ -71,6 +72,68 @@ jobs: this-job-name: build-sign-publish-chainlink continue-on-error: true + goreleaser-build-sign-publish-chainlink: + needs: [checks] + if: ${{ ! startsWith(github.ref_name, 'release/') }} + runs-on: ubuntu-20.04 + environment: build-publish + permissions: + id-token: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + + - name: Configure aws credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_ARN }} + role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }} + aws-region: ${{ secrets.AWS_REGION }} + mask-aws-account-id: true + role-session-name: goreleaser-build-sign-publish-chainlink + + - name: Build, sign, and publish image + id: goreleaser-build-sign-publish + uses: ./.github/actions/goreleaser-build-sign-publish + with: + enable-docker-publish: "true" + docker-registry: ${{ env.ECR_HOSTNAME}} + docker-image-name: ${{ env.ECR_IMAGE_NAME }} + docker-image-tag: ${{ github.ref_name }} + goreleaser-exec: ./tools/bin/goreleaser_wrapper + goreleaser-config: .goreleaser.develop.yaml + goreleaser-key: ${{ secrets.GORELEASER_KEY }} + zig-version: 0.11.0 + enable-cosign: "true" + cosign-version: 3.4.0 + cosign-password: ${{ secrets.COSIGN_PASSWORD }} + cosign-public-key: ${{ secrets.COSIGN_PUBLIC_KEY }} + cosign-private-key: ${{ secrets.COSIGN_PRIVATE_KEY }} + + - name: Output image name and digest + shell: sh + run: | + artifact_path="dist/artifacts.json" + echo "### Docker Images" | tee -a "$GITHUB_STEP_SUMMARY" + jq -r '.[] | select(.type == "Docker Image") | "`\(.goarch)-image`: \(.name)"' ${artifact_path} >> output.txt + jq -r '.[] | select(.type == "Archive") | "`\(.goarch)-digest`: \(.extra.Checksum)"' ${artifact_path} >> output.txt + while read -r line; do + echo "$line" | tee -a "$GITHUB_STEP_SUMMARY" + done < output.txt + + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 + with: + id: goreleaser-build-chainlink-publish + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: goreleaser-build-sign-publish-chainlink + continue-on-error: true + # Notify Slack channel for new git tags. slack-notify: if: github.ref_type == 'tag' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index f52b9a59746..00000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: "Build Chainlink" - -on: - pull_request: - -jobs: - build-chainlink: - runs-on: ubuntu-20.04 - if: ${{ ! startsWith(github.head_ref, 'release/') && ! startsWith(github.ref_name, 'chore/') }} - steps: - - name: Checkout repository - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - id: change - with: - predicate-quantifier: every - filters: | - changelog-only: - - 'CHANGELOG.md' - - '!common/**' - - '!contracts/**' - - '!core/**' - - '!dashboard-lib/**' - - '!fuzz/**' - - '!integration-tests/**' - - '!internal/**' - - '!operator_ui/**' - - '!plugins/**' - - '!tools/**' - - - name: Build chainlink image - if: ${{ steps.change.outputs.changelog-only == 'false' }} - uses: ./.github/actions/build-sign-publish-chainlink - with: - dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} - dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} - publish: false - sign-images: false - - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: build-chainlink - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: build-chainlink - continue-on-error: true diff --git a/.github/workflows/goreleaser-build-publish-develop.yml b/.github/workflows/goreleaser-build-publish-develop.yml deleted file mode 100644 index 835d650f181..00000000000 --- a/.github/workflows/goreleaser-build-publish-develop.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: "Build publish Chainlink develop on private ECR" - -on: - push: - branches: - - develop - -jobs: - push-chainlink-develop-goreleaser: - runs-on: - labels: ubuntu22.04-16cores-64GB - outputs: - goreleaser-metadata: ${{ steps.build-sign-publish.outputs.goreleaser-metadata }} - goreleaser-artifacts: ${{ steps.build-sign-publish.outputs.goreleaser-artifacts }} - environment: build-develop - permissions: - id-token: write - contents: read - steps: - - name: Checkout repository - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - - name: Configure aws credentials - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 - with: - role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_ARN }} - role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }} - aws-region: ${{ secrets.AWS_REGION }} - mask-aws-account-id: true - role-session-name: goreleaser-build-publish-chainlink.push-develop - - name: Build, sign, and publish image - id: build-sign-publish - uses: ./.github/actions/goreleaser-build-sign-publish - with: - enable-docker-publish: "true" - docker-registry: ${{ secrets.AWS_DEVELOP_ECR_HOSTNAME }} - enable-goreleaser-snapshot: "true" - goreleaser-exec: ./tools/bin/goreleaser_wrapper - goreleaser-config: .goreleaser.develop.yaml - goreleaser-key: ${{ secrets.GORELEASER_KEY }} - zig-version: 0.11.0 - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: goreleaser-build-publish - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: push-chainlink-develop-goreleaser - continue-on-error: true - \ No newline at end of file diff --git a/.goreleaser.develop.yaml b/.goreleaser.develop.yaml index 7fbd2aa667b..b1f65217b8f 100644 --- a/.goreleaser.develop.yaml +++ b/.goreleaser.develop.yaml @@ -4,6 +4,8 @@ project_name: chainlink env: - ZIG_EXEC={{ if index .Env "ZIG_EXEC" }}{{ .Env.ZIG_EXEC }}{{ else }}zig{{ end }} - IMAGE_PREFIX={{ if index .Env "IMAGE_PREFIX" }}{{ .Env.IMAGE_PREFIX }}{{ else }}localhost:5001{{ end }} + - IMAGE_NAME={{ if index .Env "IMAGE_NAME" }}{{ .Env.IMAGE_NAME }}{{ else }}chainlink{{ end }} + - IMAGE_TAG={{ if index .Env "IMAGE_TAG" }}{{ .Env.IMAGE_TAG }}{{ else }}develop{{ end }} - IMAGE_LABEL_DESCRIPTION="node of the decentralized oracle network, bridging on and off-chain computation" - IMAGE_LABEL_LICENSES="MIT" - IMAGE_LABEL_SOURCE="https://github.com/smartcontractkit/{{ .ProjectName }}" @@ -56,18 +58,18 @@ builds: # See https://goreleaser.com/customization/docker/ dockers: - - id: root-linux-amd64 + - id: linux-amd64 dockerfile: core/chainlink.goreleaser.Dockerfile use: buildx goos: linux goarch: amd64 extra_files: - tmp/linux_amd64/libs - - tmp/linux_amd64/plugins - tools/bin/ldd_fix build_flag_templates: - "--platform=linux/amd64" - "--pull" + - "--build-arg=CHAINLINK_USER=chainlink" - "--build-arg=COMMIT_SHA={{ .FullCommit }}" - "--label=org.opencontainers.image.created={{ .Date }}" - "--label=org.opencontainers.image.description={{ .Env.IMAGE_LABEL_DESCRIPTION }}" @@ -78,20 +80,20 @@ dockers: - "--label=org.opencontainers.image.version={{ .Env.CHAINLINK_VERSION }}" - "--label=org.opencontainers.image.url={{ .Env.IMAGE_LABEL_SOURCE }}" image_templates: - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-root-amd64" - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-root-amd64" - - id: root-linux-arm64 + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-amd64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-amd64" + - id: linux-arm64 dockerfile: core/chainlink.goreleaser.Dockerfile use: buildx goos: linux goarch: arm64 extra_files: - tmp/linux_arm64/libs - - tmp/linux_arm64/plugins - tools/bin/ldd_fix build_flag_templates: - "--platform=linux/arm64" - "--pull" + - "--build-arg=CHAINLINK_USER=chainlink" - "--build-arg=COMMIT_SHA={{ .FullCommit }}" - "--label=org.opencontainers.image.created={{ .Date }}" - "--label=org.opencontainers.image.description={{ .Env.IMAGE_LABEL_DESCRIPTION }}" @@ -102,9 +104,9 @@ dockers: - "--label=org.opencontainers.image.version={{ .Env.CHAINLINK_VERSION }}" - "--label=org.opencontainers.image.url={{ .Env.IMAGE_LABEL_SOURCE }}" image_templates: - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-root-arm64" - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-root-arm64" - - id: linux-amd64 + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-arm64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-arm64" + - id: linux-amd64-plugins dockerfile: core/chainlink.goreleaser.Dockerfile use: buildx goos: linux @@ -118,6 +120,10 @@ dockers: - "--pull" - "--build-arg=CHAINLINK_USER=chainlink" - "--build-arg=COMMIT_SHA={{ .FullCommit }}" + - "--build-arg=CL_MEDIAN_CMD=chainlink-feeds" + - "--build-arg=CL_MERCURY_CMD=chainlink-mercury" + - "--build-arg=CL_SOLANA_CMD=chainlink-solana" + - "--build-arg=CL_STARKNET_CMD=chainlink-starknet" - "--label=org.opencontainers.image.created={{ .Date }}" - "--label=org.opencontainers.image.description={{ .Env.IMAGE_LABEL_DESCRIPTION }}" - "--label=org.opencontainers.image.licenses={{ .Env.IMAGE_LABEL_LICENSES }}" @@ -127,9 +133,9 @@ dockers: - "--label=org.opencontainers.image.version={{ .Env.CHAINLINK_VERSION }}" - "--label=org.opencontainers.image.url={{ .Env.IMAGE_LABEL_SOURCE }}" image_templates: - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-amd64" - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-amd64" - - id: linux-arm64 + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-plugins-amd64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-plugins-amd64" + - id: linux-arm64-plugins dockerfile: core/chainlink.goreleaser.Dockerfile use: buildx goos: linux @@ -143,6 +149,10 @@ dockers: - "--pull" - "--build-arg=CHAINLINK_USER=chainlink" - "--build-arg=COMMIT_SHA={{ .FullCommit }}" + - "--build-arg=CL_MEDIAN_CMD=chainlink-feeds" + - "--build-arg=CL_MERCURY_CMD=chainlink-mercury" + - "--build-arg=CL_SOLANA_CMD=chainlink-solana" + - "--build-arg=CL_STARKNET_CMD=chainlink-starknet" - "--label=org.opencontainers.image.created={{ .Date }}" - "--label=org.opencontainers.image.description={{ .Env.IMAGE_LABEL_DESCRIPTION }}" - "--label=org.opencontainers.image.licenses={{ .Env.IMAGE_LABEL_LICENSES }}" @@ -152,27 +162,27 @@ dockers: - "--label=org.opencontainers.image.version={{ .Env.CHAINLINK_VERSION }}" - "--label=org.opencontainers.image.url={{ .Env.IMAGE_LABEL_SOURCE }}" image_templates: - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-arm64" - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-arm64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-plugins-arm64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-plugins-arm64" # See https://goreleaser.com/customization/docker_manifest/ docker_manifests: - - name_template: "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-root" + - name_template: "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}" image_templates: - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-root-amd64" - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-root-arm64" - - name_template: "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-root" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-amd64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-arm64" + - name_template: "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}" image_templates: - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-root-amd64" - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-root-arm64" - - name_template: "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-amd64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-arm64" + - name_template: "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-plugins" image_templates: - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-amd64" - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:develop-arm64" - - name_template: "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-plugins-amd64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:{{ .Env.IMAGE_TAG }}-plugins-arm64" + - name_template: "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-plugins" image_templates: - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-amd64" - - "{{ .Env.IMAGE_PREFIX }}/{{ .ProjectName }}-develop:sha-{{ .ShortCommit }}-arm64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-plugins-amd64" + - "{{ .Env.IMAGE_PREFIX }}/{{ .Env.IMAGE_NAME }}:sha-{{ .ShortCommit }}-plugins-arm64" # See https://goreleaser.com/customization/docker_sign/ docker_signs: @@ -185,6 +195,9 @@ checksum: snapshot: name_template: "{{ .Env.CHAINLINK_VERSION }}-{{ .ShortCommit }}" +partial: + by: target + changelog: sort: asc filters: diff --git a/core/chainlink.goreleaser.Dockerfile b/core/chainlink.goreleaser.Dockerfile index 5d172fd77e5..c35fe015cbd 100644 --- a/core/chainlink.goreleaser.Dockerfile +++ b/core/chainlink.goreleaser.Dockerfile @@ -20,8 +20,9 @@ COPY ./chainlink /usr/local/bin/ # Copy native libs if cgo is enabled COPY ./tmp/linux_${TARGETARCH}/libs /usr/local/bin/libs -# Copy plugins and enable them -COPY ./tmp/linux_${TARGETARCH}/plugins/* /usr/local/bin/ +# Copy plugins if exist and enable them +# https://stackoverflow.com/questions/70096208/dockerfile-copy-folder-if-it-exists-conditional-copy/70096420#70096420 +COPY ./tmp/linux_${TARGETARCH}/plugin[s] /usr/local/bin/ # Allow individual plugins to be enabled by supplying their path ARG CL_MEDIAN_CMD ARG CL_MERCURY_CMD diff --git a/tools/bin/goreleaser_utils b/tools/bin/goreleaser_utils index 4e1b3ffc4db..fa9553274c5 100755 --- a/tools/bin/goreleaser_utils +++ b/tools/bin/goreleaser_utils @@ -68,11 +68,13 @@ before_hook() { install_remote_plugins "linux" "amd64" "$gobin"/linux_amd64/ mkdir -p "$lib_path/linux_amd64/plugins" cp "$gobin"/linux_amd64/chainlink* "$lib_path/linux_amd64/plugins" + cp "$gobin"/chainlink* "$lib_path/linux_amd64/plugins" install_local_plugins "linux" "arm64" "$gobin"/linux_arm64/ install_remote_plugins "linux" "arm64" "$gobin"/linux_arm64/ mkdir -p "$lib_path/linux_arm64/plugins" cp "$gobin"/linux_arm64/chainlink* "$lib_path/linux_arm64/plugins" + cp "$gobin"/chainlink* "$lib_path/linux_arm64/plugins" } install_local_plugins() { @@ -94,10 +96,10 @@ get_remote_plugin_paths() { ) for plugin in "${plugins[@]}"; do - plugin_dep_name=$(echo "$plugin" | cut -d"|" -f1) - plugin_main=$(echo "$plugin" | cut -d"|" -f2) + plugin_dep_name=$(echo "$plugin" | cut -d"|" -f1) + plugin_main=$(echo "$plugin" | cut -d"|" -f2) - full_plugin_path=$(go list -m -f "{{.Dir}}" "$plugin_dep_name")"$plugin_main" + full_plugin_path=$(go list -m -f "{{.Dir}}" "$plugin_dep_name")"$plugin_main" echo "$full_plugin_path" done }