From fa8d7de469248ca39e152a6560844d984f0370c0 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 8 Aug 2023 17:45:48 +0000 Subject: [PATCH] Add test matrix to include different buildkit connectors Signed-off-by: Brian Goff --- .github/workflows/build.yml | 72 ++++++++-------------------- .github/workflows/generate-matrix.sh | 16 +++++++ scripts/ci/download-tooling.sh | 11 +++++ scripts/ci/run-functional-tests.sh | 37 ++++++++++++++ scripts/ci/setup/buildx/default | 5 ++ scripts/ci/setup/buildx/named | 5 ++ scripts/ci/setup/direct/tcp | 24 ++++++++++ scripts/ci/setup/docker/custom-unix | 31 ++++++++++++ scripts/ci/trivy_ignore.rego | 11 +++++ 9 files changed, 161 insertions(+), 51 deletions(-) create mode 100755 .github/workflows/generate-matrix.sh create mode 100755 scripts/ci/download-tooling.sh create mode 100755 scripts/ci/run-functional-tests.sh create mode 100644 scripts/ci/setup/buildx/default create mode 100644 scripts/ci/setup/buildx/named create mode 100644 scripts/ci/setup/direct/tcp create mode 100644 scripts/ci/setup/docker/custom-unix create mode 100644 scripts/ci/trivy_ignore.rego diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8e52b80f..35f40871b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,26 +79,32 @@ jobs: name: copa_edge_linux_amd64.tar.gz path: dist/linux_amd64/release/copa_edge_linux_amd64.tar.gz - name: Load test cases for patch testing - id: load-tests + id: set-matrix shell: bash - run: | - json="$(cat .github/workflows/test-images.json)" - json="${json//[$'\n'$'\r']/''}" - echo "include=$json" >> $GITHUB_OUTPUT + run: echo "test-patch-matrix=$(.github/workflows/generate-matrix.sh)" | tee -a "${GITHUB_OUTPUT}" outputs: - include: ${{ steps.load-tests.outputs.include }} + test-patch-matrix: ${{ steps.set-matrix.outputs.test-patch-matrix}} test-patch: needs: build - name: Test patch ${{ matrix.image }}:${{ matrix.tag }} + name: Test patch ${{matrix.mode }} ${{ matrix.image }}:${{ matrix.tag }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: read-all strategy: fail-fast: false - matrix: - include: ${{ fromJson(needs.build.outputs.include) }} + matrix: + include: ${{ fromJSON(needs.build.outputs.test-patch-matrix) }} steps: + - name: Download copa from build artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: copa_edge_linux_amd64.tar.gz + - name: Check out code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - name: Install required tools + shell: bash + run: ./scripts/ci/download-tooling.sh - name: Download copa from build artifacts uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: @@ -108,48 +114,12 @@ jobs: run: | tar xzf copa_edge_linux_amd64.tar.gz ./copa --version - - name: Install required tools - shell: bash - run: | - curl -sfL https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb -o trivy.deb \ - && sudo dpkg -i trivy.deb \ - && rm trivy.deb - curl -sfL https://github.com/moby/buildkit/releases/download/v${BUILDKIT_VERSION}/buildkit-v${BUILDKIT_VERSION}.linux-amd64.tar.gz -o buildkit.tar.gz \ - && sudo tar -zxvf buildkit.tar.gz -C /usr/local/ \ - && rm buildkit.tar.gz - - name: Create Trivy ignore policy - shell: bash - run: | - cat <>trivy_ignore.rego - package trivy - - import data.lib.trivy - - default ignore = false - - ignore_vulnerability_ids := { - # centos 7.6.1810 - # bind-license package version "9.11.4-26.P2.el7_9.14" does not exist - "CVE-2023-2828" - } - - ignore { - input.VulnerabilityID == ignore_vulnerability_ids[_] - } - EOF - name: Run functional test shell: bash run: | - echo "[INFO]: Patching ${{ matrix.distro }} image with: ${{ matrix.description }}" - - echo "[INFO]: Scanning image with trivy ..." - trivy image --vuln-type os --ignore-unfixed --scanners vuln -f json -o scan.json "${{ matrix.image }}:${{ matrix.tag }}@${{ matrix.digest }}" --exit-on-eol 1 --ignore-policy trivy_ignore.rego - - echo "[INFO]: Start buildkitd in the background ..." - docker run --net=host --detach --rm --privileged -p 127.0.0.1:8888:8888/tcp --name buildkitd --entrypoint buildkitd moby/buildkit:v${{ env.BUILDKIT_VERSION }} --addr tcp://0.0.0.0:8888 - - echo "[INFO]: Run copa on target ..." - ./copa patch -i "${{ matrix.image }}:${{ matrix.tag }}@${{ matrix.digest }}" -r scan.json -t "${{ matrix.tag }}-patched" -a tcp://127.0.0.1:8888 --timeout 10m - - echo "[INFO]: Rescanning patched image with same vuln DB ..." - trivy image --vuln-type os --ignore-unfixed --skip-db-update --scanners vuln "${{ matrix.image }}:${{ matrix.tag }}-patched" --exit-code 1 --exit-on-eol 1 --ignore-policy trivy_ignore.rego + echo "[INFO]: Patching ${{ matrix.distro }} image with: ${{ matrix.description}}" + COPA="$(pwd)/copa" ./scripts/ci/run-functional-tests.sh + env: + TEST_BUILDKIT_MODE: ${{ matrix.mode }} + IMAGE_REF: ${{ matrix.image }}:${{ matrix.tag }}@${{ matrix.digest }} + diff --git a/.github/workflows/generate-matrix.sh b/.github/workflows/generate-matrix.sh new file mode 100755 index 000000000..4eca3ce65 --- /dev/null +++ b/.github/workflows/generate-matrix.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +modes=() +for i in scripts/ci/setup/*; do + base="${i##*/}" + for j in "${i}"/*; do + modes+=(${base}/${j##*/}) + done +done + +modesJSON="$(jq -n '$ARGS.positional' --args ${modes[@]})" +jq --argjson modes "${modesJSON}" 'reduce .[] as $img ([]; . + reduce $modes as $mode ([]; . + [{mode: $mode} + $img] ))' < "${SCRIPT_DIR}/test-images.json" \ No newline at end of file diff --git a/scripts/ci/download-tooling.sh b/scripts/ci/download-tooling.sh new file mode 100755 index 000000000..8a3d30f40 --- /dev/null +++ b/scripts/ci/download-tooling.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +curl -sfL https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb -o trivy.deb +sudo dpkg -i trivy.deb +rm trivy.deb + +curl -sfL https://github.com/moby/buildkit/releases/download/v${BUILDKIT_VERSION}/buildkit-v${BUILDKIT_VERSION}.linux-amd64.tar.gz -o buildkit.tar.gz +sudo tar -zxvf buildkit.tar.gz -C /usr/local/ +rm buildkit.tar.gz \ No newline at end of file diff --git a/scripts/ci/run-functional-tests.sh b/scripts/ci/run-functional-tests.sh new file mode 100755 index 000000000..72ce854cf --- /dev/null +++ b/scripts/ci/run-functional-tests.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +: "${TEST_BUILDKIT_MODE:="direct/tcp"}" +: "${COPA:=copa}" +: "${COPA_FLAGS:=}" + +: "${IMAGE_NAME_ONLY="${IMAGE_REF%@*}"}" +: "${IMAGE_TAG_ONLY="${IMAGE_NAME_ONLY#*:}"}" +: "${IMAGE_TAG_PATCHED:="${IMAGE_TAG_ONLY}-patched"}" + +echo "[INFO]: Buildkit mode: ${TEST_BUILDKIT_MODE}" +echo "[INFO]: Image to patch: ${IMAGE_REF}" +echo "[INFO]: Patched image tag: ${IMAGE_TAG_PATCHED}" + +echo "[INFO]: Scanning image with trivy ..." +trivy image --vuln-type os --ignore-unfixed --scanners vuln -f json -o scan.json "${IMAGE_REF}" --exit-on-eol 1 --ignore-policy "${SCRIPT_DIR}/trivy_ignore.rego" +echo "[INFO]: Setting up buildkit with mode ${TEST_BUILDKIT_MODE} ..." + +if [ ! -f "${SCRIPT_DIR}/setup/${TEST_BUILDKIT_MODE}" ]; then + echo "[ERROR]: Unknown mode: ${TEST_BUILDKIT_MODE}" >&2 + exit 1 +fi + +. "${SCRIPT_DIR}/setup/${TEST_BUILDKIT_MODE}" + +echo "[INFO]: Run copa on target ..." +if [ -v COPA_BUILDKIT_ADDR ] && [ -n "${COPA_BUILDKIT_ADDR}" ]; then + COPA_FLAGS+="-a ${COPA_BUILDKIT_ADDR}" +fi +"${COPA}" patch -i "${IMAGE_REF}" -r scan.json -t "${IMAGE_TAG_PATCHED}" --timeout 20m ${COPA_FLAGS} + +echo "[INFO]: Rescanning patched image with same vuln DB ..." +trivy image --vuln-type os --ignore-unfixed --skip-db-update --scanners vuln "${IMAGE_TAG_PATCHED}" --exit-code 1 --exit-on-eol 1 --ignore-policy "${SCRIPT_DIR}/trivy_ignore.rego" diff --git a/scripts/ci/setup/buildx/default b/scripts/ci/setup/buildx/default new file mode 100644 index 000000000..66096b51c --- /dev/null +++ b/scripts/ci/setup/buildx/default @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +docker buildx create --use +docker buildx inspect --bootstrap +export COPA_BUILDKIT_ADDR="buildx://" \ No newline at end of file diff --git a/scripts/ci/setup/buildx/named b/scripts/ci/setup/buildx/named new file mode 100644 index 000000000..55f9b3b15 --- /dev/null +++ b/scripts/ci/setup/buildx/named @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +docker buildx create --name testpatch +docker buildx inspect --bootstrap testpatch +export COPA_BUILDKIT_ADDR="buildx://testpatch" diff --git a/scripts/ci/setup/direct/tcp b/scripts/ci/setup/direct/tcp new file mode 100644 index 000000000..f9088a659 --- /dev/null +++ b/scripts/ci/setup/direct/tcp @@ -0,0 +1,24 @@ +#!/usr/bin/env sh + + +: "${BUILDKIT_PORT:=30321}" +: "${BUILDKIT_VERSION=0.12.0}" + +_buildkit_direct_tcp_id="$(docker run --detach --rm --privileged -p 127.0.0.1::${BUILDKIT_PORT}/tcp --entrypoint buildkitd "moby/buildkit:v$BUILDKIT_VERSION" --addr tcp://0.0.0.0:${BUILDKIT_PORT})" +_buildkitd_tcp_addr="$(docker port ${_buildkit_direct_tcp_id} ${BUILDKIT_PORT})" +export COPA_BUILDKIT_ADDR="tcp://${_buildkitd_tcp_addr}" + +_cleanup() { + docker rm -f "${_buildkit_direct_tcp_id}" +} + +trap '_cleanup' EXIT + +_check_buildkitd_tcp() { + buildctl --addr ${COPA_BUILDKIT_ADDR} debug info +} + +echo "[INFO] Wait for buildkitd to be ready @ ${COPA_BUILDKIT_ADDR}" +while ! _check_buildkitd_tcp; do + sleep 1 +done \ No newline at end of file diff --git a/scripts/ci/setup/docker/custom-unix b/scripts/ci/setup/docker/custom-unix new file mode 100644 index 000000000..0aeb178e1 --- /dev/null +++ b/scripts/ci/setup/docker/custom-unix @@ -0,0 +1,31 @@ +#!/usr/bin/env sh + +# dockerd requires containerd snapshotter support to be enabled otherwise required features for buildkit are disabled. +docker build -t dind -< /etc/docker/daemon.json +ENTRYPOINT ["dockerd"] +EOF + +: "${DOCKER_DIND_VOLUME:="copa-docker-dind"}" + +sock_dir="$(mktemp -d)" + +docker_custom_unix_id="$(docker run -d --privileged --mount=type=bind,source="${sock_dir}",target=/run --mount=type=volume,source="${DOCKER_DIND_VOLUME}",target=/var/lib/docker dind --group "$(id -g)")" + +_cleanup() { + docker rm -f "$docker_custom_unix_id" + sudo rm -rf "${sock_dir}" +} + +trap '_cleanup' EXIT + +_check_docker_dind() { + docker -H "unix://${sock_dir}/docker.sock" info +} + +while ! _check_docker_dind; do + check_docker_dind || sleep 1 +done + +export COPA_BUILDKIT_ADDR="docker://unix://${sock_dir}/docker.sock" \ No newline at end of file diff --git a/scripts/ci/trivy_ignore.rego b/scripts/ci/trivy_ignore.rego new file mode 100644 index 000000000..55636a4db --- /dev/null +++ b/scripts/ci/trivy_ignore.rego @@ -0,0 +1,11 @@ +package trivy + +import data.lib.trivy + +default ignore = false + +ignore_vulnerability_ids := { + # centos 7.6.1810 + # bind-license package version "9.11.4-26.P2.el7_9.14" does not exist + "CVE-2023-2828" +} \ No newline at end of file