From c6baf03962cc2b238d47a6ff21d5c6dd0abad88b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 3 Oct 2024 07:52:46 -0600 Subject: [PATCH] idaholab/Malcolm#585, fall back to alternative Zeek .deb download URL --- ...dgehog-iso-build-docker-wrap-push-ghcr.yml | 1 + ...ehog-raspi-build-docker-wrap-push-ghcr.yml | 1 + .../workflows/zeek-build-and-push-ghcr.yml | 1 + Dockerfiles/zeek.Dockerfile | 1 + hedgehog-iso/build.sh | 2 +- hedgehog-iso/build_via_vagrant.sh | 1 + hedgehog-raspi/build_via_vagrant.sh | 1 + hedgehog-raspi/sensor_install.sh | 2 +- scripts/build.sh | 4 +- shared/bin/zeek-deb-download.sh | 50 +++++++++++++------ 10 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml index f88e9a402..6a16d715e 100644 --- a/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml @@ -129,6 +129,7 @@ jobs: echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt echo "${{ secrets.MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/maxmind_url.txt + echo "${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/zeek_url.txt echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot echo "BUILD_JOBS=2" > ./shared/environment.chroot diff --git a/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml b/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml index 0d879c261..21c46dbfd 100644 --- a/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml @@ -86,6 +86,7 @@ jobs: echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt echo "${{ secrets.MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/maxmind_url.txt + echo "${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/zeek_url.txt echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot echo "BUILD_JOBS=2" > ./shared/environment.chroot diff --git a/.github/workflows/zeek-build-and-push-ghcr.yml b/.github/workflows/zeek-build-and-push-ghcr.yml index 4cec0de26..26fac60b3 100644 --- a/.github/workflows/zeek-build-and-push-ghcr.yml +++ b/.github/workflows/zeek-build-and-push-ghcr.yml @@ -105,6 +105,7 @@ jobs: MALCOLM_VERSION=${{ steps.extract_malcolm_version.outputs.mversion }} BUILD_DATE=${{ steps.generate_build_timestamp.outputs.btimestamp }} VCS_REVISION=${{ steps.extract_commit_sha.outputs.sha }} + ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }} push: true provenance: false platforms: ${{ matrix.platform }} diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index a384965d2..f6e002139 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -35,6 +35,7 @@ USER root # for download and install ARG ZEEK_VERSION=7.0.2-0 ENV ZEEK_VERSION $ZEEK_VERSION +ARG ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="" # put Zeek and Spicy in PATH ENV ZEEK_DIR "/opt/zeek" diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh index 61e750b72..4c8234ad2 100755 --- a/hedgehog-iso/build.sh +++ b/hedgehog-iso/build.sh @@ -190,7 +190,7 @@ if [ -d "$WORKDIR" ]; then "https://github.com/arkime/arkime/releases/download/v${ARKIME_VER}/arkime_${ARKIME_VER}-1.debian12_amd64.deb" # download Zeek .deb packages - bash "$SCRIPT_PATH/shared/bin/zeek-deb-download.sh" -o ./config/packages.chroot/ + bash "$SCRIPT_PATH/shared/bin/zeek-deb-download.sh" -o ./config/packages.chroot/ -f "$SCRIPT_PATH/shared/zeek_url.txt" # reclaim some space docker system prune --volumes --force diff --git a/hedgehog-iso/build_via_vagrant.sh b/hedgehog-iso/build_via_vagrant.sh index bd66f5ea5..c15457f83 100755 --- a/hedgehog-iso/build_via_vagrant.sh +++ b/hedgehog-iso/build_via_vagrant.sh @@ -100,6 +100,7 @@ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-co [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt [[ ${#MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/maxmind_url.txt +[[ ${#ZEEK_DEB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/zeek_url.txt [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot trap cleanup_shared_and_docs EXIT diff --git a/hedgehog-raspi/build_via_vagrant.sh b/hedgehog-raspi/build_via_vagrant.sh index fac209dea..643f4378b 100755 --- a/hedgehog-raspi/build_via_vagrant.sh +++ b/hedgehog-raspi/build_via_vagrant.sh @@ -99,6 +99,7 @@ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-co [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt [[ ${#MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/maxmind_url.txt +[[ ${#ZEEK_DEB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/zeek_url.txt [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot trap cleanup_shared_and_docs EXIT diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index 0aef0e19a..ad12d4b1c 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -197,7 +197,7 @@ build_yara_src() { build_zeek() { # install zeek from debs from OpenSUSE mkdir -p /tmp/zeek-debs - /bin/bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-debs + /bin/bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-debs -f "$SHARED_DIR/zeek_url.txt" dpkg -i /tmp/zeek-debs/*.deb } diff --git a/scripts/build.sh b/scripts/build.sh index ff21bebd8..050128dbc 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -122,9 +122,9 @@ fi # build the image(s) DOCKER_COMPOSE_COMMAND="${DOCKER_COMPOSE_BIN[@]} --profile malcolm -f "$CONFIG_FILE"" if [[ $CONFIRMATION =~ ^[Yy] ]]; then - $DOCKER_COMPOSE_COMMAND --progress=plain build --force-rm --no-cache --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@" + $DOCKER_COMPOSE_COMMAND --progress=plain build --force-rm --no-cache --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@" else - $DOCKER_COMPOSE_COMMAND --progress=plain build --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@" + $DOCKER_COMPOSE_COMMAND --progress=plain build --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@" fi # we're going to do some validation that some things got pulled/built correctly diff --git a/shared/bin/zeek-deb-download.sh b/shared/bin/zeek-deb-download.sh index c527938d3..ca755ac9e 100755 --- a/shared/bin/zeek-deb-download.sh +++ b/shared/bin/zeek-deb-download.sh @@ -7,12 +7,18 @@ command -v dpkg >/dev/null 2>&1 && ARCH="$(dpkg --print-architecture)" || ARCH=a DISTRO=Debian_12 OUTPUT_DIR=/tmp ZEEK_VERSION=7.0.2-0 +PRESERVE_HIERARCHY=false +ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-} +ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE=${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE:-} -while getopts a:d:o:vz: opts; do +while getopts a:d:f:ho:u:vz: opts; do case ${opts} in a) ARCH=${OPTARG} ;; d) DISTRO=${OPTARG} ;; + f) ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE=${OPTARG} ;; + h) PRESERVE_HIERARCHY=true ;; o) OUTPUT_DIR=${OPTARG} ;; + u) ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${OPTARG} ;; v) VERBOSE=1 ;; z) ZEEK_VERSION=${OPTARG} ;; esac @@ -23,23 +29,37 @@ if [[ -n $VERBOSE ]]; then set -x fi -URL_PREFIX="https://downloadcontentcdn.opensuse.org/repositories/security:/zeek/${DISTRO}" -URLS=( - "${URL_PREFIX}/${ARCH}/libbroker-dev_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeek-core-dev_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeek-core_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeek-spicy-dev_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeek_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeekctl_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/all/zeek-client_${ZEEK_VERSION}_all.deb" - "${URL_PREFIX}/all/zeek-zkg_${ZEEK_VERSION}_all.deb" - "${URL_PREFIX}/all/zeek-btest_${ZEEK_VERSION}_all.deb" - "${URL_PREFIX}/all/zeek-btest-data_${ZEEK_VERSION}_all.deb" +if [[ -z "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL}" ]] && [[ -f "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE}" ]]; then + ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="$(head -n 1 "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE}")" +fi + +URL_PREFIXES=( + "https://downloadcontentcdn.opensuse.org/repositories/security:/zeek" +) +[[ -n "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" ]] && URL_PREFIXES+=( "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" ) + +URL_SUFFIXES=( + "${DISTRO}/${ARCH}/libbroker-dev_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeek-core-dev_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeek-core_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeek-spicy-dev_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeek_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeekctl_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/all/zeek-client_${ZEEK_VERSION}_all.deb" + "${DISTRO}/all/zeek-zkg_${ZEEK_VERSION}_all.deb" + "${DISTRO}/all/zeek-btest_${ZEEK_VERSION}_all.deb" + "${DISTRO}/all/zeek-btest-data_${ZEEK_VERSION}_all.deb" ) pushd "$OUTPUT_DIR" >/dev/null 2>&1 -for URL in ${URLS[@]}; do - curl -fsSL -O -J "${URL}" +for URL_SUFFIX in ${URL_SUFFIXES[@]}; do + [[ "$PRESERVE_HIERARCHY" == "true" ]] && OUTPUT_DIR_REL="$(dirname "$URL_SUFFIX")" || OUTPUT_DIR_REL=. + mkdir -p "$OUTPUT_DIR_REL" + pushd "$OUTPUT_DIR_REL" >/dev/null 2>&1 + for URL_PREFIX in ${URL_PREFIXES[@]}; do + curl -fsSL -O -J "${URL_PREFIX%/}/${URL_SUFFIX}" && break + done + popd >/dev/null 2>&1 done popd >/dev/null 2>&1