Add test matrix to include different buildkit connectors

Signed-off-by: Brian Goff <[email protected]>
project-copacetic · Aug 11, 2023 · fa8d7de · fa8d7de
1 parent 9ab74ab
commit fa8d7de
Show file tree

Hide file tree

Showing 9 changed files with 161 additions and 51 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -79,26 +79,32 @@ jobs:
           name: copa_edge_linux_amd64.tar.gz
           path: dist/linux_amd64/release/copa_edge_linux_amd64.tar.gz
       - name: Load test cases for patch testing
-        id: load-tests
+        id: set-matrix
         shell: bash
-        run: |
-          json="$(cat .github/workflows/test-images.json)"
-          json="${json//[$'\n'$'\r']/''}"
-          echo "include=$json" >> $GITHUB_OUTPUT
+        run: echo "test-patch-matrix=$(.github/workflows/generate-matrix.sh)" | tee -a "${GITHUB_OUTPUT}"
     outputs:
-      include: ${{ steps.load-tests.outputs.include }}
+      test-patch-matrix: ${{ steps.set-matrix.outputs.test-patch-matrix}}
 
   test-patch:
     needs: build
-    name: Test patch ${{ matrix.image }}:${{ matrix.tag }}
+    name: Test patch ${{matrix.mode }} ${{ matrix.image }}:${{ matrix.tag }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     permissions: read-all
     strategy:
       fail-fast: false
-      matrix:
-        include: ${{ fromJson(needs.build.outputs.include) }}
+      matrix: 
+        include: ${{ fromJSON(needs.build.outputs.test-patch-matrix) }}
     steps:
+      - name: Download copa from build artifacts
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: copa_edge_linux_amd64.tar.gz
+      - name: Check out code
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - name: Install required tools
+        shell: bash
+        run: ./scripts/ci/download-tooling.sh
       - name: Download copa from build artifacts
         uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
@@ -108,48 +114,12 @@ jobs:
         run: |
           tar xzf copa_edge_linux_amd64.tar.gz
           ./copa --version
-      - name: Install required tools
-        shell: bash
-        run: |
-          curl -sfL https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb -o trivy.deb \
-            && sudo dpkg -i trivy.deb \
-            && rm trivy.deb
-          curl -sfL https://github.com/moby/buildkit/releases/download/v${BUILDKIT_VERSION}/buildkit-v${BUILDKIT_VERSION}.linux-amd64.tar.gz -o buildkit.tar.gz \
-            && sudo tar -zxvf buildkit.tar.gz -C /usr/local/ \
-            && rm buildkit.tar.gz
-      - name: Create Trivy ignore policy
-        shell: bash
-        run: |
-          cat <<EOF >>trivy_ignore.rego
-          package trivy
-
-          import data.lib.trivy
-
-          default ignore = false
-
-          ignore_vulnerability_ids := {
-            # centos 7.6.1810
-            # bind-license package version "9.11.4-26.P2.el7_9.14" does not exist
-            "CVE-2023-2828"
-          }
-
-          ignore {
-            input.VulnerabilityID == ignore_vulnerability_ids[_]
-          }
-          EOF
       - name: Run functional test
         shell: bash
         run: |
-          echo "[INFO]: Patching ${{ matrix.distro }} image with: ${{ matrix.description }}"
-
-          echo "[INFO]: Scanning image with trivy ..."
-          trivy image --vuln-type os --ignore-unfixed --scanners vuln -f json -o scan.json "${{ matrix.image }}:${{ matrix.tag }}@${{ matrix.digest }}" --exit-on-eol 1 --ignore-policy trivy_ignore.rego
-
-          echo "[INFO]: Start buildkitd in the background ..."
-          docker run --net=host --detach --rm --privileged -p 127.0.0.1:8888:8888/tcp --name buildkitd --entrypoint buildkitd moby/buildkit:v${{ env.BUILDKIT_VERSION }} --addr tcp://0.0.0.0:8888
-
-          echo "[INFO]: Run copa on target ..."
-          ./copa patch -i "${{ matrix.image }}:${{ matrix.tag }}@${{ matrix.digest }}" -r scan.json -t "${{ matrix.tag }}-patched" -a tcp://127.0.0.1:8888 --timeout 10m
-
-          echo "[INFO]: Rescanning patched image with same vuln DB ..."
-          trivy image --vuln-type os --ignore-unfixed --skip-db-update --scanners vuln "${{ matrix.image }}:${{ matrix.tag }}-patched" --exit-code 1 --exit-on-eol 1 --ignore-policy trivy_ignore.rego
+            echo "[INFO]: Patching ${{ matrix.distro }} image with: ${{ matrix.description}}"
+            COPA="$(pwd)/copa" ./scripts/ci/run-functional-tests.sh
+        env:
+          TEST_BUILDKIT_MODE: ${{ matrix.mode }}
+          IMAGE_REF: ${{ matrix.image }}:${{ matrix.tag }}@${{ matrix.digest }}
+
diff --git a/.github/workflows/generate-matrix.sh b/.github/workflows/generate-matrix.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+modes=()
+for i in scripts/ci/setup/*; do
+    base="${i##*/}"
+    for j in "${i}"/*; do
+        modes+=(${base}/${j##*/})
+    done
+done
+
+modesJSON="$(jq -n '$ARGS.positional' --args ${modes[@]})"
+jq --argjson modes "${modesJSON}" 'reduce .[] as $img ([]; . + reduce $modes as $mode ([]; . + [{mode: $mode} + $img] ))' < "${SCRIPT_DIR}/test-images.json"
diff --git a/scripts/ci/download-tooling.sh b/scripts/ci/download-tooling.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+curl -sfL https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb -o trivy.deb
+sudo dpkg -i trivy.deb
+rm trivy.deb
+
+curl -sfL https://github.com/moby/buildkit/releases/download/v${BUILDKIT_VERSION}/buildkit-v${BUILDKIT_VERSION}.linux-amd64.tar.gz -o buildkit.tar.gz
+sudo tar -zxvf buildkit.tar.gz -C /usr/local/
+rm buildkit.tar.gz
diff --git a/scripts/ci/run-functional-tests.sh b/scripts/ci/run-functional-tests.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+: "${TEST_BUILDKIT_MODE:="direct/tcp"}"
+: "${COPA:=copa}"
+: "${COPA_FLAGS:=}"
+
+: "${IMAGE_NAME_ONLY="${IMAGE_REF%@*}"}"
+: "${IMAGE_TAG_ONLY="${IMAGE_NAME_ONLY#*:}"}"
+: "${IMAGE_TAG_PATCHED:="${IMAGE_TAG_ONLY}-patched"}"
+
+echo "[INFO]: Buildkit mode: ${TEST_BUILDKIT_MODE}"
+echo "[INFO]: Image to patch: ${IMAGE_REF}"
+echo "[INFO]: Patched image tag: ${IMAGE_TAG_PATCHED}"
+
+echo "[INFO]: Scanning image with trivy ..."
+trivy image --vuln-type os --ignore-unfixed --scanners vuln -f json -o scan.json "${IMAGE_REF}" --exit-on-eol 1 --ignore-policy "${SCRIPT_DIR}/trivy_ignore.rego"
+echo "[INFO]: Setting up buildkit with mode ${TEST_BUILDKIT_MODE} ..."
+
+if [ ! -f "${SCRIPT_DIR}/setup/${TEST_BUILDKIT_MODE}" ]; then
+    echo "[ERROR]: Unknown mode: ${TEST_BUILDKIT_MODE}" >&2
+    exit 1
+fi
+
+. "${SCRIPT_DIR}/setup/${TEST_BUILDKIT_MODE}"
+
+echo "[INFO]: Run copa on target ..."
+if [ -v COPA_BUILDKIT_ADDR ] && [ -n "${COPA_BUILDKIT_ADDR}" ]; then
+    COPA_FLAGS+="-a ${COPA_BUILDKIT_ADDR}"
+fi
+"${COPA}" patch -i "${IMAGE_REF}" -r scan.json -t "${IMAGE_TAG_PATCHED}" --timeout 20m ${COPA_FLAGS}
+
+echo "[INFO]: Rescanning patched image with same vuln DB ..."
+trivy image --vuln-type os --ignore-unfixed --skip-db-update --scanners vuln "${IMAGE_TAG_PATCHED}" --exit-code 1 --exit-on-eol 1 --ignore-policy "${SCRIPT_DIR}/trivy_ignore.rego"
diff --git a/scripts/ci/setup/buildx/default b/scripts/ci/setup/buildx/default
@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+docker buildx create --use
+docker buildx inspect --bootstrap
+export COPA_BUILDKIT_ADDR="buildx://"
diff --git a/scripts/ci/setup/buildx/named b/scripts/ci/setup/buildx/named
@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+docker buildx create --name testpatch
+docker buildx inspect --bootstrap testpatch
+export COPA_BUILDKIT_ADDR="buildx://testpatch"
diff --git a/scripts/ci/setup/direct/tcp b/scripts/ci/setup/direct/tcp
@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+
+
+: "${BUILDKIT_PORT:=30321}"
+: "${BUILDKIT_VERSION=0.12.0}"
+
+_buildkit_direct_tcp_id="$(docker run --detach --rm --privileged -p 127.0.0.1::${BUILDKIT_PORT}/tcp  --entrypoint buildkitd "moby/buildkit:v$BUILDKIT_VERSION" --addr tcp://0.0.0.0:${BUILDKIT_PORT})"
+_buildkitd_tcp_addr="$(docker port ${_buildkit_direct_tcp_id} ${BUILDKIT_PORT})"
+export COPA_BUILDKIT_ADDR="tcp://${_buildkitd_tcp_addr}"
+
+_cleanup() {
+    docker rm -f "${_buildkit_direct_tcp_id}"
+}
+
+trap '_cleanup' EXIT
+
+_check_buildkitd_tcp() {
+    buildctl --addr ${COPA_BUILDKIT_ADDR} debug info
+}
+
+echo "[INFO] Wait for buildkitd to be ready @ ${COPA_BUILDKIT_ADDR}"
+while ! _check_buildkitd_tcp; do
+    sleep 1
+done
diff --git a/scripts/ci/setup/docker/custom-unix b/scripts/ci/setup/docker/custom-unix
@@ -0,0 +1,31 @@
+#!/usr/bin/env sh
+
+# dockerd requires containerd snapshotter support to be enabled otherwise required features for buildkit are disabled.
+docker build -t dind -<<EOF
+FROM docker:24.0-dind
+RUN mkdir -p /etc/docker && echo '{"features": { "containerd-snapshotter": true }}' > /etc/docker/daemon.json
+ENTRYPOINT ["dockerd"]
+EOF
+
+: "${DOCKER_DIND_VOLUME:="copa-docker-dind"}"
+
+sock_dir="$(mktemp -d)"
+
+docker_custom_unix_id="$(docker run -d --privileged --mount=type=bind,source="${sock_dir}",target=/run --mount=type=volume,source="${DOCKER_DIND_VOLUME}",target=/var/lib/docker dind --group "$(id -g)")"
+
+_cleanup() {
+    docker rm -f "$docker_custom_unix_id"
+    sudo rm -rf "${sock_dir}"
+}
+
+trap '_cleanup' EXIT
+
+_check_docker_dind() {
+    docker -H "unix://${sock_dir}/docker.sock" info
+}
+
+while ! _check_docker_dind; do
+    check_docker_dind || sleep 1
+done
+
+export COPA_BUILDKIT_ADDR="docker://unix://${sock_dir}/docker.sock"
diff --git a/scripts/ci/trivy_ignore.rego b/scripts/ci/trivy_ignore.rego
@@ -0,0 +1,11 @@
+package trivy
+
+import data.lib.trivy
+
+default ignore = false
+
+ignore_vulnerability_ids := {
+    # centos 7.6.1810
+    # bind-license package version "9.11.4-26.P2.el7_9.14" does not exist
+    "CVE-2023-2828"
+}