bazel: use remote caching (#1456)

* bazel: add configuration for remote caching
* ci: enable bazel remote caching for building binaries
* ci: use bazel directly when building go binaries
* ci: enable cache for most build steps
* dev-docs: document remote caching

.bazelrc

@@ -38,3 +38,12 @@ build:linux_arm64_static --platforms @zig_sdk//libc_aware/platform:linux_arm64_m
 
 # bazel config to explicitly disable stamping (hide version information at build time)
 build:nostamp --nostamp --workspace_status_command=
+
+# bazel config to use remote cache
+build:remote_cache --bes_results_url=https://app.buildbuddy.io/invocation/
+build:remote_cache --bes_backend=grpcs://remote.buildbuddy.io
+build:remote_cache --remote_cache=grpcs://remote.buildbuddy.io
+build:remote_cache --remote_timeout=3600
+build:remote_cache --experimental_remote_build_event_upload=minimal
+build:remote_cache --experimental_remote_cache_compression
+build:remote_cache_readonly --noremote_upload_local_results # Uploads logs & artifacts without writing to cache

.github/actions/build_bootstrapper/action.yml

@@ -4,7 +4,7 @@ description: Build the Constellation bootstrapper binary
 inputs:
   outputPath:
     description: "Output path of the binary"
-    default: "./bootstrapper"
+    default: "./build/bootstrapper"
     required: true
 
 # Linux runner only (Docker required)
@@ -13,10 +13,16 @@ runs:
   steps:
     - name: Build the bootstrapper
       shell: bash
+      env:
+        OUTPUT_PATH: ${{ inputs.outputPath }}
       run: |
         echo "::group::Build the bootstrapper"
-        mkdir -p build && cd build
-        cmake -DBAZEL:STRING=bazelisk ..
-        make bootstrapper
-        mv -n bootstrapper "${{ inputs.outputPath }}"
+        mkdir -p "$(dirname "${OUTPUT_PATH}")"
+        label=//bootstrapper/cmd/bootstrapper:bootstrapper_linux_amd64
+        bazel build "${label}"
+        repository_root=$(git rev-parse --show-toplevel)
+        out_rel=$(bazel cquery --output=files "${label}")
+        out_loc="$(realpath "${repository_root}/${out_rel}")"
+        cp "${out_loc}" "${OUTPUT_PATH}"
+        chmod +w "${OUTPUT_PATH}"
         echo "::endgroup::"

.github/actions/build_cdbg/action.yml

@@ -10,15 +10,28 @@ inputs:
     description: "Build CLI for this architecture. [amd64, arm64]"
     required: true
     default: "amd64"
+  outputPath:
+    description: "Output path of the binary"
+    default: "./build/cdbg"
+    required: false
 
 runs:
   using: "composite"
   steps:
     - name: Build cdbg
       shell: bash
+      env:
+        GOOS: ${{ inputs.targetOS }}
+        GOARCH: ${{ inputs.targetArch }}
+        OUTPUT_PATH: ${{ inputs.outputPath }}
       run: |
         echo "::group::Build cdbg"
-        mkdir -p build && cd build
-        cmake ..
-        GOOS=${{ inputs.targetOS }} GOARCH=${{ inputs.targetArch }} make cdbg
+        mkdir -p "$(dirname "${OUTPUT_PATH}")"
+        label="//debugd/cmd/cdbg:cdbg_${GOOS}_${GOARCH}"
+        bazel build "${label}"
+        repository_root=$(git rev-parse --show-toplevel)
+        out_rel=$(bazel cquery --output=files "${label}")
+        out_loc="$(realpath "${repository_root}/${out_rel}")"
+        cp "${out_loc}" "${OUTPUT_PATH}"
+        chmod +w "${OUTPUT_PATH}"
         echo "::endgroup::"

.github/actions/build_cli/action.yml

@@ -1,6 +1,6 @@
 name: Build CLI
 description: |
-  Runs cmake and cli make target in build folder. Optionally, Sigstore tools
+  Builds CLI. Optionally, Sigstore tools
   are used to sign CLI when inputs are provided. A draft release is published
   when run on v* tag.
 inputs:
@@ -28,6 +28,9 @@ inputs:
     description: "Password for Cosign private key"
     required: false
     default: ""
+  outputPath:
+    description: "Output path of the binary"
+    required: false
 runs:
   using: "composite"
   steps:
@@ -39,20 +42,28 @@ runs:
 
     - name: Build CLI
       shell: bash
+      env:
+        GOOS: ${{ inputs.targetOS }}
+        GOARCH: ${{ inputs.targetArch }}
+        OUTPUT_PATH: ${{ inputs.outputPath || format('./build/constellation-{0}-{1}', inputs.targetOS, inputs.targetArch) }}
       run: |
         echo "::group::Build CLI"
-        mkdir -p build
-        cd build
+        mkdir -p "$(dirname "${OUTPUT_PATH}")"
         if [ ${{ inputs.enterpriseCLI }} == 'true' ]
         then
-          cmake -DBAZEL:STRING=bazelisk -DCLI_BUILD_TAGS:STRING=enterprise ..
+          cli_variant=enterprise
         else
-          cmake -DBAZEL:STRING=bazelisk ..
+          cli_variant=oss
         fi
-        GOOS=${{ inputs.targetOS }} GOARCH=${{ inputs.targetArch }} make cli
-        cp constellation constellation-${{ inputs.targetOS }}-${{ inputs.targetArch }}
-        echo "$(pwd)" >> $GITHUB_PATH
-        export PATH="$PATH:$(pwd)"
+        label="//cli:cli_${cli_variant}_${GOOS}_${GOARCH}"
+        bazel build "${label}"
+        repository_root=$(git rev-parse --show-toplevel)
+        out_rel=$(bazel cquery --output=files "${label}")
+        out_loc="$(realpath "${repository_root}/${out_rel}")"
+        cp "${out_loc}" "${OUTPUT_PATH}"
+        chmod +w "${OUTPUT_PATH}"
+        echo "$(dirname "${OUTPUT_PATH}")" >> $GITHUB_PATH
+        export PATH="$PATH:$(dirname "${OUTPUT_PATH}")"
         echo "::endgroup::"
 
     # TODO: Replace with https://github.com/sigstore/sigstore-installer/tree/initial
@@ -80,15 +91,15 @@ runs:
         COSIGN_PUBLIC_KEY: ${{ inputs.cosignPublicKey }}
         COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }}
         COSIGN_PASSWORD: ${{ inputs.cosignPassword }}
+        OUTPUT_PATH: ${{ inputs.outputPath || format('./build/constellation-{0}-{1}', inputs.targetOS, inputs.targetArch) }}
       run: |
-        SIGN_TARGET=constellation-${{ inputs.targetOS }}-${{ inputs.targetArch }}
         echo "$COSIGN_PUBLIC_KEY" > cosign.pub
         # Enabling experimental mode also publishes signature to Rekor
-        COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY ${SIGN_TARGET} > ${SIGN_TARGET}.sig
+        COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY "${OUTPUT_PATH}" > "${OUTPUT_PATH}.sig"
         # Verify - As documentation & check
         # Local Signature (input: artifact, key, signature)
-        cosign verify-blob --key cosign.pub --signature ${SIGN_TARGET}.sig ${SIGN_TARGET}
+        cosign verify-blob --key cosign.pub --signature "${OUTPUT_PATH}.sig" "${OUTPUT_PATH}"
         # Transparency Log Signature (input: artifact, key)
-        uuid=$(rekor-cli search --artifact ${SIGN_TARGET} | tail -n 1)
+        uuid=$(rekor-cli search --artifact "${OUTPUT_PATH}" | tail -n 1)
         sig=$(rekor-cli get --uuid=$uuid --format=json | jq -r .Body.HashedRekordObj.signature.content)
-        cosign verify-blob --key cosign.pub --signature <(echo $sig) ${SIGN_TARGET}
+        cosign verify-blob --key cosign.pub --signature <(echo $sig) "${OUTPUT_PATH}"

.github/actions/build_debugd/action.yml

@@ -4,7 +4,7 @@ description: Build the Constellation debugd binary
 inputs:
   outputPath:
     description: "Output path of the binary"
-    default: "./debugd"
+    default: "./build/debugd"
     required: true
 
 # Linux runner only (homedir trick does not work on macOS, required for private runner)
@@ -13,14 +13,16 @@ runs:
   steps:
     - name: Build debugd
       shell: bash
+      env:
+        OUTPUT_PATH: ${{ inputs.outputPath }}
       run: |
         echo "::group::Build debugd"
-        homedir="$(getent passwd $(id -u) | cut -d ":" -f 6)"
-        mkdir -p build && cd build
-        cmake -DBAZEL:STRING=bazelisk ..
-        export GOCACHE=${homedir}/.cache/go-build
-        export GOPATH=${homedir}/go
-        export GOMODCACHE=${homedir}/.cache/go-mod
-        make debugd
-        mv -n debugd "${{ inputs.outputPath }}"
+        mkdir -p "$(dirname "${OUTPUT_PATH}")"
+        label=//debugd/cmd/debugd:debugd_linux_amd64
+        bazel build "${label}"
+        repository_root=$(git rev-parse --show-toplevel)
+        out_rel=$(bazel cquery --output=files "${label}")
+        out_loc="$(realpath "${repository_root}/${out_rel}")"
+        cp "${out_loc}" "${OUTPUT_PATH}"
+        chmod +w "${OUTPUT_PATH}"
         echo "::endgroup::"

.github/actions/build_disk_mapper/action.yml

@@ -4,7 +4,7 @@ description: Build the Constellation disk-mapper binary
 inputs:
   outputPath:
     description: "Output path of the binary"
-    default: "./disk-mapper"
+    default: "./build/disk-mapper"
     required: true
 
 # Linux runner only (Docker required)
@@ -13,10 +13,16 @@ runs:
   steps:
     - name: Build the disk-mapper
       shell: bash
+      env:
+        OUTPUT_PATH: ${{ inputs.outputPath }}
       run: |
         echo "::group::Build the disk-mapper"
-        mkdir -p build && cd build
-        cmake -DBAZEL:STRING=bazelisk ..
-        make disk-mapper
-        mv -n disk-mapper "${{ inputs.outputPath }}"
+        mkdir -p "$(dirname "${OUTPUT_PATH}")"
+        label="//disk-mapper/cmd:disk-mapper_linux_amd64"
+        bazel build "${label}"
+        repository_root=$(git rev-parse --show-toplevel)
+        out_rel=$(bazel cquery --output=files "${label}")
+        out_loc="$(realpath "${repository_root}/${out_rel}")"
+        cp "${out_loc}" "${OUTPUT_PATH}"
+        chmod +w "${OUTPUT_PATH}"
         echo "::endgroup::"

.github/actions/build_measurement_reader/action.yml

@@ -4,7 +4,7 @@ description: Build the Constellation measurement-reader binary
 inputs:
   outputPath:
     description: "Output path of the binary"
-    default: "./measurement-reader"
+    default: "./build/measurement-reader"
     required: true
 
 # Linux runner only (Docker required)
@@ -13,10 +13,16 @@ runs:
   steps:
     - name: Build the measurement-reader
       shell: bash
+      env:
+        OUTPUT_PATH: ${{ inputs.outputPath }}
       run: |
         echo "::group::Build the measurement-reader"
-        mkdir -p build && cd build
-        cmake ..
-        make measurement-reader
-        mv -n measurement-reader "${{ inputs.outputPath }}"
+        mkdir -p "$(dirname "${OUTPUT_PATH}")"
+        label="//measurement-reader/cmd:measurement-reader_linux_amd64"
+        bazel build "${label}"
+        repository_root=$(git rev-parse --show-toplevel)
+        out_rel=$(bazel cquery --output=files "${label}")
+        out_loc="$(realpath "${repository_root}/${out_rel}")"
+        cp "${out_loc}" "${OUTPUT_PATH}"
+        chmod +w "${OUTPUT_PATH}"
         echo "::endgroup::"

.github/actions/build_upgrade_agent/action.yml

@@ -4,7 +4,7 @@ description: Build the Constellation upgrade-agent binary
 inputs:
   outputPath:
     description: "Output path of the binary"
-    default: "./upgrade-agent"
+    default: "./build/upgrade-agent"
     required: true
 
 # Linux runner only (Docker required)
@@ -13,10 +13,16 @@ runs:
   steps:
     - name: Build the upgrade-agent
       shell: bash
+      env:
+        OUTPUT_PATH: ${{ inputs.outputPath }}
       run: |
         echo "::group::Build the upgrade-agent"
-        mkdir -p build && cd build
-        cmake -DBAZEL:STRING=bazelisk ..
-        make upgrade-agent
-        mv -n upgrade-agent "${{ inputs.outputPath }}"
+        mkdir -p "$(dirname "${OUTPUT_PATH}")"
+        label="//upgrade-agent/cmd:upgrade_agent_linux_amd64"
+        bazel build "${label}"
+        repository_root=$(git rev-parse --show-toplevel)
+        out_rel=$(bazel cquery --output=files "${label}")
+        out_loc="$(realpath "${repository_root}/${out_rel}")"
+        cp "${out_loc}" "${OUTPUT_PATH}"
+        chmod +w "${OUTPUT_PATH}"
         echo "::endgroup::"

.github/actions/e2e_mini/action.yml

@@ -11,6 +11,9 @@ inputs:
   azureTenantID:
     description: "Azure tenant to use for login with OIDC"
     required: true
+  buildBuddyApiKey:
+    description: "BuildBuddy API key for caching Bazel artifacts"
+    required: true
 
 runs:
   using: "composite"
@@ -25,10 +28,17 @@ runs:
       with:
         go-version: "1.20.2"
 
+    - name: Setup bazel
+      uses: ./.github/actions/setup_bazel
+      with:
+        useCache: "true"
+        buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
+
     - name: Build Constellation CLI
       uses: ./.github/actions/build_cli
       with:
         enterpriseCLI: true
+        outputPath: build/constellation
 
     - name: MiniConstellation E2E
       shell: bash

.github/actions/e2e_test/action.yml

@@ -71,6 +71,9 @@ inputs:
   sonobuoyTestSuiteCmd:
     description: "The sonobuoy test suite to run."
     required: false
+  buildBuddyApiKey:
+    description: "BuildBuddy API key for caching Bazel artifacts"
+    required: true
 
 outputs:
   kubeconfig:
@@ -104,12 +107,19 @@ runs:
         echo "hostOS=$(go env GOOS)" >> $GITHUB_OUTPUT
         echo "hostArch=$(go env GOARCH)" >> $GITHUB_OUTPUT
 
+    - name: Setup bazel
+      uses: ./.github/actions/setup_bazel
+      with:
+        useCache: "true"
+        buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
+
     - name: Build CLI
       uses: ./.github/actions/build_cli
       with:
         targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
         targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
         enterpriseCLI: ${{ inputs.keepMeasurements }}
+        outputPath: "build/constellation"
 
     - name: Build the bootstrapper
       id: build-bootstrapper

.github/actions/setup_bazel/action.yml

@@ -0,0 +1,55 @@
+name: Setup bazel
+description: Setup Bazel for CI builds and tests
+
+inputs:
+  useCache:
+    description: "Cache Bazel artifacts. Use 'true' to enable with rw, 'readonly' to download, and 'false' to disable."
+    default: "false"
+    required: true
+  buildBuddyApiKey:
+    description: "BuildBuddy API key for caching Bazel artifacts"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check inputs
+      shell: bash
+      run: |
+        echo "::group::Check inputs"
+        if [[ "${{ inputs.useCache }}" != "true" && "${{ inputs.useCache }}" != "readonly" && "${{ inputs.useCache }}" != "false" ]]; then
+          echo "Invalid value for 'useCache' input: '${{ inputs.useCache }}'. Must be 'true', 'readonly', or 'false'."
+          exit 1
+        fi
+        if [[ "${{ inputs.useCache }}" == "true" || "${{ inputs.useCache }}" == "readonly" ]] && [[ -z "${{ inputs.buildBuddyApiKey }}" ]]; then
+          echo "BuildBuddy API key is required when cache is enabled."
+          exit 1
+        fi
+        echo "::endgroup::"
+    - name: Configure Bazel
+      shell: bash
+      if: inputs.useCache == 'true' || inputs.useCache == 'readonly'
+      env:
+        BUILDBUDDY_ORG_API_KEY: ${{ inputs.buildBuddyApiKey }}
+      run: |
+        echo "::group::Configure Bazel"
+        cat <<EOF >> ~/.bazelrc
+        build --bes_results_url=https://app.buildbuddy.io/invocation/
+        build --bes_backend=grpcs://remote.buildbuddy.io
+        build --remote_cache=grpcs://remote.buildbuddy.io
+        build --remote_timeout=3600
+        build --experimental_remote_build_event_upload=minimal
+        build --experimental_remote_cache_compression
+        build --remote_header=x-buildbuddy-api-key=${BUILDBUDDY_ORG_API_KEY}
+        EOF
+        echo "::endgroup::"
+    - name: Configure Bazel (readonly)
+      shell: bash
+      if: inputs.useCache == 'readonly'
+      run: |
+        echo "::group::Configure Bazel (readonly)"
+        echo "build --remote_upload_local_results=false" >> ~/.bazelrc
+        echo "::endgroup::"
+    - name: Check bazel version
+      shell: bash
+      run: bazel version