workflows: support 1.8 branch builds plus MacOS (and Windows for stag…

…ing) (#5248) * workflows: rename to Linux only Signed-off-by: Patrick Stephens <[email protected]> * workflows: add reusable job for Windows build Signed-off-by: Patrick Stephens <[email protected]> * workflows: call Windows CI in staging build Signed-off-by: Patrick Stephens <[email protected]> * workflows: update container build job to handle 1.8 branches Signed-off-by: Patrick Stephens <[email protected]> * workflows: add MacOS build Signed-off-by: Patrick Stephens <[email protected]> * workflows: linting and other updates Signed-off-by: Patrick Stephens <[email protected]> * workflows: linting fix Signed-off-by: Patrick Stephens <[email protected]> * workflows: add permissions to new workflows Signed-off-by: Patrick Stephens <[email protected]> * workflows: temporary change to invoke reusable workflows on this branch, REVERT BEFORE MERGE Signed-off-by: Patrick Stephens <[email protected]> * workflows: fix names Signed-off-by: Patrick Stephens <[email protected]> * workflows: add build matrix support for branches Signed-off-by: Patrick Stephens <[email protected]> * workflows: update PRs to use Windows job Signed-off-by: Patrick Stephens <[email protected]> * workflows: support legacy td-agent-bit in Windows builds Signed-off-by: Patrick Stephens <[email protected]> * workflows: fix review comment Signed-off-by: Patrick Stephens <[email protected]> * workflows: revert temporary branch change for testing Signed-off-by: Patrick Stephens <[email protected]> * workflows: document needs for build matrix action Signed-off-by: Patrick Stephens <[email protected]>
fluent · Apr 7, 2022 · 2ecaa7b · 2ecaa7b
1 parent 09b7315
commit 2ecaa7b
Show file tree

Hide file tree

Showing 10 changed files with 395 additions and 138 deletions.
diff --git a/.github/actions/generate-package-build-matrix/action.yaml b/.github/actions/generate-package-build-matrix/action.yaml
@@ -2,23 +2,46 @@
 name: Composite action to generate the matrix of targets to build packages for.
 description: Remove any duplication of this information so we only have to update in one place.
 
+# Remember to add any new checks and target creation required.
+# For example, when 2.0 comes out we should detect it and add any target changes.
+
 inputs:
   target:
     description: Override to build a single target for debug/test only.
     required: false
-  fluent-bit-version:
-    description: The version of Fluent Bit to build.
-    required: false
-    default: 'master'
+    default: ''
+  ref:
+    description: The commit, tag or branch of Fluent Bit to checkout for building we then use to determine version for.
+    required: true
 outputs:
   build-matrix:
     description: The build matrix we have created.
     value: ${{ steps.set-matrix.outputs.matrix }}
 runs:
   using: "composite"
   steps:
-    - name: master/1.9 targets
-      if: inputs.fluent-bit-version != '1.8'
+    - name: Checkout code for version check
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ inputs.ref }}
+        path: version-check
+
+    - name: Determine target type
+      # TODO: probably switch to a sentinel file with the version in rather than checking for specific changes, or git branch tests.
+      id: determine-build-type
+      run: |
+        ls -l
+        BUILD_TYPE="1.8"
+        if [[ -d "packaging" ]]; then
+          BUILD_TYPE="1.9"
+        fi
+        echo "Detected type: $BUILD_TYPE"
+        echo ::set-output name=BUILD_TYPE::$BUILD_TYPE
+      shell: bash
+      working-directory: version-check
+
+    - name: 1.9 targets
+      if: steps.determine-build-type.outputs.BUILD_TYPE == '1.9'
       run: |
         matrix=$((
           echo '{ "distro" : ['
@@ -29,19 +52,11 @@ runs:
           echo '"raspbian/buster", "raspbian/bullseye"'
           echo ']}'
         ) | jq -c .)
-        if [ -n "${{ inputs.target || '' }}" ]; then
-          echo "Overriding matrix to build: ${{ inputs.target }}"
-          matrix=$((
-            echo '{ "distro" : ['
-            echo '"${{ inputs.target }}"'
-            echo ']}'
-          ) | jq -c .)
-        fi
         echo "MATRIX=$matrix" >> $GITHUB_ENV
       shell: bash
 
     - name: 1.8 targets
-      if: inputs.fluent-bit-version == '1.8'
+      if: steps.determine-build-type.outputs.BUILD_TYPE == '1.8'
       run: |
         matrix=$((
           echo '{ "distro" : ['
@@ -52,6 +67,12 @@ runs:
           echo '"raspbian/buster"'
           echo ']}'
         ) | jq -c .)
+        echo "MATRIX=$matrix" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Manual override of target
+      if: inputs.target != ''
+      run: |
         if [ -n "${{ inputs.target || '' }}" ]; then
           echo "Overriding matrix to build: ${{ inputs.target }}"
           matrix=$((

diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -58,6 +58,13 @@ For some reason this is not automatically done via permission inheritance or sim
 2. Once this is working locally, you should then be able to set up action permissions for the repository. If you already have a package no need to push a test one.
 3. Go to `https://github.com/users/USER/packages/container/fluent-bit/settings` and ensure the repository has access to `Write`.
 
+## Version-specific targets
+
+Each major version (e.g. 1.8 & 1.9) supports different targets to build for, e.g. 1.9 includes a CentOS 8 target and 1.8 has some other legacy targets.
+
+This is all handled by the [build matrix generation composite action](../actions/generate-package-build-matrix/action.yaml) so make sure to update appropriately.
+The build matrix is then fed into the reusable job that builds packages which will then fire for the appropriate targets.
+
 ## Releases
 
 Currently the process is as follows:

diff --git a/.github/workflows/build-master-packages.yaml b/.github/workflows/build-master-packages.yaml
@@ -42,7 +42,7 @@ jobs:
 
   master-build-packages:
     needs: master-build-generate-matrix
-    uses: fluent/fluent-bit/.github/workflows/call-build-packages.yaml@master
+    uses: fluent/fluent-bit/.github/workflows/call-build-linux-packages.yaml@master
     with:
       version: master
       ref: master

diff --git a/.github/workflows/call-build-images.yaml b/.github/workflows/call-build-images.yaml
@@ -44,26 +44,54 @@ on:
         description: If the Cosign key requires a password then specify here, otherwise not required.
         required: false
 jobs:
+  call-build-legacy-check:
+    # Determine if this is a 1.8 type of build which is different
+    name: Extract any supporting metadata
+    outputs:
+      build-type: ${{ steps.determine-build-type.outputs.BUILD_TYPE }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Determine build type
+        id: determine-build-type
+        run: |
+          BUILD_TYPE="1.8"
+          if [[ -f "dockerfiles/Dockerfile" ]]; then
+            BUILD_TYPE="modern"
+          fi
+          echo "Detected type: $BUILD_TYPE"
+          echo ::set-output name=BUILD_TYPE::$BUILD_TYPE
+        shell: bash
+
   call-build-images:
     name: Multiarch container images to GHCR
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
+    needs:
+      - call-build-legacy-check
     permissions:
       contents: read
       packages: write
     steps:
-      - name: Checkout code
-        if: inputs.ref != '1.8'
+      - name: Checkout code for modern style builds
+        if: needs.call-build-legacy-check.outputs.build-type != '1.8'
         uses: actions/checkout@v3
         with:
           ref: ${{ inputs.ref }}
 
-      - name: Checkout the docker build repo
-        if: inputs.ref == '1.8'
+      - name: Checkout the docker build repo for legacy builds
+        if: needs.call-build-legacy-check.outputs.build-type == '1.8'
         uses: actions/checkout@v3
         with:
           repository: fluent/fluent-bit-docker-image
-          ref: '1.8'
+          ref: '1.8' # Fixed to this branch
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -88,7 +116,7 @@ jobs:
             raw,latest
 
       - name: Build the production images
-        if: inputs.ref != '1.8'
+        if: needs.call-build-legacy-check.outputs.build-type != '1.8'
         uses: docker/build-push-action@v2
         with:
           file: ./dockerfiles/Dockerfile
@@ -111,7 +139,7 @@ jobs:
             raw,latest-debug
 
       - name: Build the debug multi-arch images
-        if: inputs.ref != '1.8'
+        if: needs.call-build-legacy-check.outputs.build-type != '1.8'
         uses: docker/build-push-action@v2
         with:
           file: ./dockerfiles/Dockerfile
@@ -126,7 +154,7 @@ jobs:
             FLB_NIGHTLY_BUILD=${{ inputs.unstable }}
 
       - name: Build the legacy image
-        if: inputs.ref == '1.8'
+        if: needs.call-build-legacy-check.outputs.build-type == '1.8'
         uses: docker/build-push-action@v2
         with:
           file: Dockerfile.x86_64
@@ -140,10 +168,12 @@ jobs:
             FLB_TARBALL=https://github.com/fluent/fluent-bit/tarball/${{ inputs.ref }}
 
   call-build-images-generate-schema:
-    if: inputs.ref != '1.8'
+    needs:
+      - call-build-legacy-check
+      - call-build-images
+    if: needs.call-build-legacy-check.outputs.build-type != '1.8'
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
-    needs: call-build-images
     permissions:
       contents: read
       packages: read

diff --git a/.github/workflows/call-build-packages.yaml → .../workflows/call-build-linux-packages.yaml b/.github/workflows/call-build-packages.yaml → .../workflows/call-build-linux-packages.yaml
@@ -46,25 +46,54 @@ on:
         required: false
 
 jobs:
+  call-build-legacy-check:
+    # Determine if this is a 1.8 type of build which is different
+    name: Extract any supporting metadata
+    outputs:
+      build-type: ${{ steps.determine-build-type.outputs.BUILD_TYPE }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Determine build type
+        id: determine-build-type
+        run: |
+          BUILD_TYPE="1.8"
+          if [[ -f "dockerfiles/Dockerfile" ]]; then
+            BUILD_TYPE="modern"
+          fi
+          echo "Detected type: $BUILD_TYPE"
+          echo ::set-output name=BUILD_TYPE::$BUILD_TYPE
+        shell: bash
 
   # We build both master/1.9 and 1.8 style packages in a single job as it is simpler to keep all the custom
   # code and workflow together. We need to run the dependent job after this as well.
-  call-build-packages:
+  call-build-linux-packages:
     name: ${{ matrix.distro }} package build and stage to S3
     environment: ${{ inputs.environment }}
     runs-on: ubuntu-latest
+    needs:
+      - call-build-legacy-check
+    permissions:
+      contents: read
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(inputs.build_matrix) }}
     steps:
       - name: Checkout code
-        if: inputs.ref != '1.8'
+        if: needs.call-build-legacy-check.outputs.build-type != '1.8'
         uses: actions/checkout@v3
         with:
           ref: ${{ inputs.ref }}
 
-      - name: Checkout packaging code
-        if: inputs.ref == '1.8'
+      - name: Checkout legacy packaging code
+        if: needs.call-build-legacy-check.outputs.build-type == '1.8'
         uses: actions/checkout@v3
         with:
           repository: fluent/fluent-bit-packaging
@@ -85,7 +114,7 @@ jobs:
           flags: 'g'
 
       - name: fluent-bit - ${{ matrix.distro }} artifacts
-        if: inputs.ref != '1.8'
+        if: needs.call-build-legacy-check.outputs.build-type != '1.8'
         run: |
           ./build.sh
         env:
@@ -98,7 +127,7 @@ jobs:
         working-directory: packaging
 
       - name: td-agent-bit - ${{ matrix.distro }} artifacts
-        if: inputs.ref != '1.8'
+        if: needs.call-build-legacy-check.outputs.build-type != '1.8'
         run: |
           ./build.sh
         env:
@@ -111,7 +140,7 @@ jobs:
         working-directory: packaging
 
       - name: Legacy td-agent-bit - ${{ matrix.distro }} artifacts
-        if: inputs.ref == '1.8'
+        if: needs.call-build-legacy-check.outputs.build-type == '1.8'
         run: |
           ./build.sh -v $FLB_VERSION -b $FLB_BRANCH -d $FLB_DISTRO
         env:
@@ -153,7 +182,7 @@ jobs:
 
       - name: Push packages to S3
         # Only upload for staging
-        if: ${{ inputs.environment == 'staging' }}
+        if: inputs.environment == 'staging'
         # Make sure not to do a --delete on sync as it will remove the other architecture
         run: |
           if [ -z "${{ steps.get-target-info.outputs.target }}" ]; then
@@ -172,15 +201,15 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.secret_access_key }}
           AWS_S3_BUCKET: ${{ secrets.bucket }}
 
-  call-build-packages-repo:
+  call-build-linux-packages-repo:
     name: Create repo metadata in S3
     # Only upload for staging
-    if: ${{ inputs.environment == 'staging' }}
+    if: inputs.environment == 'staging'
     # Need to use 18.04 as 20.04 has no createrepo available
     runs-on: ubuntu-18.04
     environment: ${{ inputs.environment }}
     needs:
-      - call-build-packages
+      - call-build-linux-packages
     steps:
       - name: Install dependencies
         run: |