From 15ed8585aceeefab2ea6695c18938fb719d195e0 Mon Sep 17 00:00:00 2001 From: Pat Date: Tue, 21 Dec 2021 14:40:35 +0000 Subject: [PATCH] workflows: reusable package build for master and staging (#4498) * workflows: reusable package build for master & staging Signed-off-by: Patrick Stephens * workflows: resolve linting issues Signed-off-by: Patrick Stephens --- .github/workflows/build-master-packages.yaml | 67 +++---- .github/workflows/call-build-packages.yaml | 186 ++++++++----------- .github/workflows/staging-build.yaml | 26 ++- 3 files changed, 126 insertions(+), 153 deletions(-) diff --git a/.github/workflows/build-master-packages.yaml b/.github/workflows/build-master-packages.yaml index afce78a8beb..406e88e2c25 100644 --- a/.github/workflows/build-master-packages.yaml +++ b/.github/workflows/build-master-packages.yaml @@ -2,53 +2,32 @@ on: push: branches: - master + workflow_dispatch: name: Build packages for master jobs: - build-distro-packages: - name: build packages - strategy: - max-parallel: 48 - fail-fast: true - matrix: - distro: [ ubuntu/16.04, ubuntu/18.04, ubuntu/20.04, debian/buster ] - - runs-on: [ ubuntu-latest ] #self-hosted, Linux, X64, packet-builder] + master-build-generate-matrix: + name: Staging build matrix + runs-on: ubuntu-latest + outputs: + build-matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - name: Setup environment + # Set up the list of target to build so we can pass the JSON to the reusable job + - id: set-matrix run: | - sudo apt-get update - sudo apt-get install --yes qemu binfmt-support qemu-user-static qemu-utils qemu-efi-aarch64 qemu-system-arm docker.io containerd runc - sudo systemctl unmask docker && sudo systemctl start docker - docker run --rm --privileged --name qemu multiarch/qemu-user-static:register --reset - - - uses: frabert/replace-string-action@master - id: formatted_distro - with: - pattern: '(.*)\/(.*)$' - string: "${{ matrix.distro }}" - replace-with: '$1-$2' - flags: 'g' - - - uses: actions/checkout@v2 - with: - repository: fluent/fluent-bit-packaging - fetch-depth: 1 - path: packaging - - - name: Build the distro artifacts - run: ./build.sh -v master -d ${{ env.distro }} -b master - env: - distro: ${{ matrix.distro }} - working-directory: packaging + matrix=$(( + echo '{ "distro" : [ "debian/bullseye", "ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04" ]}' + ) | jq -c .) + echo $matrix + echo $matrix| jq . + echo "::set-output name=matrix::$matrix" + shell: bash - - name: Store the master package artifacts - uses: actions/upload-artifact@v2 - with: - name: packages-${{env.release}}-${{env.bucket-name}} - path: | - packaging/packages/${{env.distro}}/${{env.release}}/**/* - env: - bucket-name: ${{ steps.formatted_distro.outputs.replaced }} - release: master - distro: ${{ matrix.distro }} + master-build-packages: + needs: master-build-generate-matrix + uses: fluent/fluent-bit/.github/workflows/call-build-packages.yaml@master + with: + version: master + build_matrix: ${{ needs.master-build-generate-matrix.outputs.build-matrix }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/call-build-packages.yaml b/.github/workflows/call-build-packages.yaml index 0f9df25f028..ac929ecadc6 100644 --- a/.github/workflows/call-build-packages.yaml +++ b/.github/workflows/call-build-packages.yaml @@ -8,6 +8,10 @@ on: description: The version of the Fluent Bit to build. type: string required: true + build_matrix: + description: The build targets to produce as a JSON matrix. + type: string + required: true environment: description: The Github environment to run this workflow on. type: string @@ -18,16 +22,16 @@ on: required: true bucket: description: The name of the S3 (US-East) bucket to push packages into. - required: true + required: false access_key_id: description: The S3 access key id for the bucket. - required: true + required: false secret_access_key: description: The S3 secret access key for the bucket. - required: true + required: false gpg_private_key: description: The GPG key to use for signing the packages. - required: true + required: false jobs: call-build-packages: @@ -36,120 +40,86 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: true - matrix: - distro: [ amazonlinux/2, amazonlinux/2.arm64v8, - centos/7, centos/7.arm64v8, centos/8, centos/8.arm64v8, - debian/stretch, debian/stretch.arm64v8, debian/buster, debian/buster.arm64v8, debian/bullseye, debian/bullseye.arm64v8, - ubuntu/16.04, ubuntu/18.04, ubuntu/20.04, ubuntu/18.04.arm64v8, ubuntu/20.04.arm64v8, - raspbian/buster ] - include: - - distro: ubuntu/18.04 - target: ubuntu/bionic - - - distro: amazonlinux/2 - target: amazonlinux/2/ - - - distro: amazonlinux/2.arm64v8 - target: amazonlinux/2/ - - - distro: centos/7 - target: centos/7/ - - - distro: centos/7.arm64v8 - target: centos/7/ - - - distro: centos/8 - target: centos/8/ - - - distro: centos/8.arm64v8 - target: centos/8/ - - - distro: debian/stretch - target: debian/stretch/ - - - distro: debian/stretch.arm64v8 - target: debian/stretch/ - - - distro: debian/buster - target: debian/buster/ - - - distro: debian/buster.arm64v8 - target: debian/buster/ - - - distro: debian/bullseye - target: debian/bullseye/ - - - distro: debian/bullseye.arm64v8 - target: debian/bullseye/ - - - distro: ubuntu/16.04 - target: ubuntu/xenial/ - - - distro: ubuntu/18.04.arm64v8 - target: ubuntu/bionic/ - - - distro: ubuntu/20.04 - target: ubuntu/focal/ - - - distro: ubuntu/20.04.arm64v8 - target: ubuntu/focal/ - - - distro: raspbian/buster - target: raspbian/buster/ + matrix: ${{ fromJSON(inputs.build_matrix) }} steps: + - name: Checkout code + uses: actions/checkout@v2 - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - - uses: frabert/replace-string-action@master - id: formatted_distro - with: - pattern: '(.*)\/(.*)$' - string: "${{ matrix.distro }}" - replace-with: '$1-$2' - flags: 'g' + - uses: frabert/replace-string-action@master + id: formatted_distro + with: + pattern: '(.*)\/(.*)$' + string: "${{ matrix.distro }}" + replace-with: '$1-$2' + flags: 'g' - - name: Build the ${{ matrix.distro }} artifacts - run: | - ./build.sh -v "${{ inputs.version }}" -d "${{ matrix.distro }}" - env: - FLB_OUT_DIR: staging - working-directory: packaging + - name: Build the ${{ matrix.distro }} artifacts + run: | + ./build.sh -v "${{ inputs.version }}" -d "${{ matrix.distro }}" + env: + FLB_OUT_DIR: staging + working-directory: packaging - - name: Push packages to S3 - # Make sure not to do a --delete on sync as it will remove the other architecture - run: | - if [ -n "${AWS_S3_ENDPOINT}" ]; then - ENDPOINT="--endpoint-url ${AWS_S3_ENDPOINT}" - fi - aws --region "$AWS_REGION" s3 sync "${SOURCE_DIR}" "s3://${AWS_S3_BUCKET}/${DEST_DIR}" --follow-symlinks --no-progress ${ENDPOINT} - env: - SOURCE_DIR: "packaging/packages/${{ matrix.distro }}/${{ inputs.version }}/staging/" - DEST_DIR: "${{ inputs.version }}/${{ matrix.target }}" # No extra slashes as they affect the URL to download - AWS_REGION: "us-east-1" - AWS_ACCESS_KEY_ID: ${{ secrets.access_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.secret_access_key }} - AWS_S3_BUCKET: ${{ secrets.bucket }} - # To use with Minio locally (or update to whatever endpoint you want) - # AWS_S3_ENDPOINT: http://localhost:9000 + - name: Upload the ${{ matrix.distro }} artifacts + uses: actions/upload-artifact@v2 + with: + name: packages-${{ inputs.version }}-${{ steps.formatted_distro.outputs.replaced }} + path: packaging/packages/staging/ + if-no-files-found: error + + - name: Retrieve target info for repo creation + id: get-target-info + # Remove any .arm648 suffix + # For ubuntu map to codename using the disto-info list (CSV) + run: | + sudo apt-get update + sudo apt-get install -y distro-info + TARGET=${DISTRO%*.arm64v8} + if [[ "$TARGET" == "ubuntu/"* ]]; then + UBUNTU_CODENAME=$(cut -d ',' -f 1,3 < "/usr/share/distro-info/ubuntu.csv"|grep "${TARGET##*/}"|cut -d ',' -f 2) + if [[ -n "$UBUNTU_CODENAME" ]]; then + TARGET="ubuntu/$UBUNTU_CODENAME" + else + echo "Unable to extract codename for $DISTRO" + exit 1 + fi + fi + echo ::set-output name=echo ::set-output name=target::$TARGET + env: + DISTRO: ${{ matrix.distro }} + shell: bash - - name: Upload the ${{ matrix.distro }} artifacts - if: always() - uses: actions/upload-artifact@v2 - with: - name: ${{ steps.formatted_distro.outputs.replaced }}-package - path: packaging/packages/ - if-no-files-found: error + - name: Push packages to S3 + # Only upload for staging + if: ${{ inputs.environment == 'staging' }} + # Make sure not to do a --delete on sync as it will remove the other architecture + run: | + if [ -n "${AWS_S3_ENDPOINT}" ]; then + ENDPOINT="--endpoint-url ${AWS_S3_ENDPOINT}" + fi + aws --region "$AWS_REGION" s3 sync "${SOURCE_DIR}" "s3://${AWS_S3_BUCKET}/${DEST_DIR}" --follow-symlinks --no-progress ${ENDPOINT} + env: + SOURCE_DIR: "packaging/packages/${{ matrix.distro }}/${{ inputs.version }}/staging/" + DEST_DIR: "${{ inputs.version }}/${{ steps.get-target-info.outputs.target }}/" + AWS_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: ${{ secrets.access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.secret_access_key }} + AWS_S3_BUCKET: ${{ secrets.bucket }} + # To use with Minio locally (or update to whatever endpoint you want) + # AWS_S3_ENDPOINT: http://localhost:9000 call-build-packages-repo: name: Create repo metadata in S3 - runs-on: ubuntu-18.04 # no createrepo otherwise + # Only upload for staging + if: ${{ inputs.environment == 'staging' }} + # Need to use 18.04 as 20.04 has no createrepo available + runs-on: ubuntu-18.04 environment: ${{ inputs.environment }} needs: call-build-packages steps: diff --git a/.github/workflows/staging-build.yaml b/.github/workflows/staging-build.yaml index 8a8972e9943..df8b3385905 100644 --- a/.github/workflows/staging-build.yaml +++ b/.github/workflows/staging-build.yaml @@ -67,11 +67,35 @@ jobs: cosign_private_key: ${{ secrets.COSIGN_PRIVATE_KEY }} cosign_private_key_password: ${{ secrets.COSIGN_PASSWORD }} + staging-build-generate-matrix: + name: Staging build matrix + runs-on: ubuntu-latest + outputs: + build-matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + # Set up the list of target to build so we can pass the JSON to the reusable job + - id: set-matrix + run: | + matrix=$(( + echo '{ "distro" : [' + echo '"amazonlinux/2", "amazonlinux/2.arm64v8",' + echo '"centos/7", "centos/7.arm64v8", "centos/8", "centos/8.arm64v8",' + echo '"debian/stretch", "debian/stretch.arm64v8", "debian/buster", "debian/buster.arm64v8", "debian/bullseye", "debian/bullseye.arm64v8",' + echo '"ubuntu/16.04", "ubuntu/18.04", "ubuntu/20.04", "ubuntu/18.04.arm64v8", "ubuntu/20.04.arm64v8"' + echo '"raspbian/buster", "raspbian/bullseye"' + echo ']}' + ) | jq -c .) + echo $matrix + echo $matrix| jq . + echo "::set-output name=matrix::$matrix" + shell: bash + staging-build-packages: - needs: staging-build-get-version + needs: [ staging-build-get-version, staging-build-generate-matrix ] uses: fluent/fluent-bit/.github/workflows/call-build-packages.yaml@master with: version: ${{ needs.staging-build-get-version.outputs.version }} + build_matrix: ${{ needs.staging-build-generate-matrix.outputs.build-matrix }} environment: staging secrets: token: ${{ secrets.GITHUB_TOKEN }}