From 1f3c282168ec0b8a6229b30f695730d6cb3771c7 Mon Sep 17 00:00:00 2001 From: Pat Date: Mon, 14 Mar 2022 15:03:46 +0000 Subject: [PATCH] packaging: build from source rather than tar (#5050) Signed-off-by: Patrick Stephens --- .github/workflows/build-master-packages.yaml | 1 - .github/workflows/call-build-packages.yaml | 8 --- packaging/build.sh | 72 ++++---------------- packaging/distros/amazonlinux/Dockerfile | 17 ++--- packaging/distros/centos/Dockerfile | 17 ++--- packaging/distros/debian/Dockerfile | 18 ++--- packaging/distros/raspbian/Dockerfile | 18 ++--- packaging/distros/ubuntu/Dockerfile | 18 ++--- packaging/local-build-all.sh | 9 +-- 9 files changed, 35 insertions(+), 143 deletions(-) diff --git a/.github/workflows/build-master-packages.yaml b/.github/workflows/build-master-packages.yaml index 9e2bd6e447f..a7f539a19d3 100644 --- a/.github/workflows/build-master-packages.yaml +++ b/.github/workflows/build-master-packages.yaml @@ -46,7 +46,6 @@ jobs: with: version: master ref: master - branch: master build_matrix: ${{ needs.master-build-generate-matrix.outputs.build-matrix }} secrets: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/call-build-packages.yaml b/.github/workflows/call-build-packages.yaml index 4b20e7831c3..231c6d1afd9 100644 --- a/.github/workflows/call-build-packages.yaml +++ b/.github/workflows/call-build-packages.yaml @@ -20,10 +20,6 @@ on: description: The Github environment to run this workflow on. type: string required: false - branch: - description: The optional source code branch to use for building against. - type: string - required: false unstable: description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add. type: string @@ -81,10 +77,8 @@ jobs: run: | ./build.sh env: - FLB_VERSION: ${{ inputs.version }} FLB_DISTRO: ${{ matrix.distro }} FLB_OUT_DIR: staging - FLB_BRANCH: ${{ inputs.branch }} FLB_TD: "Off" FLB_NIGHTLY_BUILD: ${{ inputs.unstable }} CMAKE_INSTALL_PREFIX: /opt/fluent-bit/ @@ -94,10 +88,8 @@ jobs: run: | ./build.sh env: - FLB_VERSION: ${{ inputs.version }} FLB_DISTRO: ${{ matrix.distro }} FLB_OUT_DIR: staging - FLB_BRANCH: ${{ inputs.branch }} FLB_TD: "On" FLB_NIGHTLY_BUILD: ${{ inputs.unstable }} CMAKE_INSTALL_PREFIX: /opt/td-agent-bit/ diff --git a/packaging/build.sh b/packaging/build.sh index ad3ac14d0ea..07b50d973df 100755 --- a/packaging/build.sh +++ b/packaging/build.sh @@ -1,16 +1,12 @@ #!/bin/bash -set -eu +set -eux # Never rely on PWD so we can invoke from anywhere SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # Allow us to specify in the caller or pass variables -FLB_BRANCH=${FLB_BRANCH:-} -FLB_PREFIX=${FLB_PREFIX:-} -FLB_VERSION=${FLB_VERSION:-} FLB_DISTRO=${FLB_DISTRO:-} FLB_OUT_DIR=${FLB_OUT_DIR:-} -FLB_TARGZ=${FLB_TARGZ:-} FLB_NIGHTLY_BUILD=${FLB_NIGHTLY_BUILD:-} FLB_JEMALLOC=${FLB_JEMALLOC:-On} @@ -21,30 +17,20 @@ while getopts "v:d:b:t:o:" option do case "${option}" in - v) FLB_VERSION=${OPTARG};; d) FLB_DISTRO=${OPTARG};; - b) FLB_BRANCH=${OPTARG};; - t) FLB_TARGZ=${OPTARG};; o) FLB_OUT_DIR=${OPTARG};; *) echo "Unknown option";; esac done -if [ -z "$FLB_VERSION" ] || [ -z "$FLB_DISTRO" ]; then +if [ -z "$FLB_DISTRO" ]; then echo "$@" - echo "Usage: build.sh -v VERSION -d DISTRO" - echo " ^ ^ " - echo " | 1.9.0 | ubuntu/20.04" + echo "Usage: build.sh -d DISTRO" + echo " ^ " + echo " | ubuntu/20.04" exit 1 fi -if [ -z "$FLB_BRANCH" ]; then - # The standard tags have a v prefix but we may want to build others - if curl -sL --output /dev/null --head --fail --retry 3 "http://github.com/fluent/fluent-bit/archive/v$FLB_VERSION.zip" ; then - FLB_PREFIX="v" - fi -fi - # Prepare output directory if [ -n "$FLB_OUT_DIR" ]; then out_dir=$FLB_OUT_DIR @@ -52,17 +38,14 @@ else out_dir=$(date '+%Y-%m-%d-%H_%M_%S') fi -volume="$SCRIPT_DIR/packages/$FLB_DISTRO/$FLB_VERSION/$out_dir/" +volume="$SCRIPT_DIR/packages/$FLB_DISTRO/$out_dir/" mkdir -p "$volume" # Info -echo "FLB_PREFIX => $FLB_PREFIX" -echo "FLB_VERSION => $FLB_VERSION" echo "FLB_DISTRO => $FLB_DISTRO" -echo "FLB_SRC => $FLB_TARGZ" echo "FLB_OUT_DIR => $FLB_OUT_DIR" -MAIN_IMAGE="flb-$FLB_VERSION-$FLB_DISTRO" +MAIN_IMAGE="flb-$FLB_DISTRO" # We either have a specific Dockerfile in the distro directory or we have a generic multi-stage one for all # of the same OS type: @@ -82,39 +65,13 @@ if [[ ! -f "$IMAGE_CONTEXT_DIR/Dockerfile" ]]; then exit 1 fi -echo "IMAGE_CONTEXT_DIR => $IMAGE_CONTEXT_DIR" - -# Create sources directory if it does not exist -mkdir -p "$IMAGE_CONTEXT_DIR/sources" - -# Set tarball as an argument (./build.sh VERSION DISTRO/CODENAME -t something.tar.gz) -if [ -n "$FLB_TARGZ" ]; then - # Check if we have a local tarball - if [[ ! -f "$FLB_TARGZ" ]]; then - echo "Unable to find tarball: $FLB_TARGZ" - exit 1 - fi - - # Copy tarball - cp "$FLB_TARGZ" "$IMAGE_CONTEXT_DIR/sources/" - # Set build argument (ensure we strip off any path) - FLB_ARG="$FLB_ARG --build-arg FLB_SRC=$(basename "$FLB_TARGZ")" -else - # Check we have a valid remote source URL - FLB_SOURCE_URL="http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip" - if ! curl -sL --output /dev/null --head --fail --retry 3 "$FLB_SOURCE_URL" ; then - echo "Unable to download source from URL:$FLB_SOURCE_URL " - exit 1 - fi -fi - # CMake configuration variables, override via environment rather than parameters CMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-/opt/td-agent-bit/} FLB_TD=${FLB_TD:-On} +echo "IMAGE_CONTEXT_DIR => $IMAGE_CONTEXT_DIR" echo "CMAKE_INSTALL_PREFIX => $CMAKE_INSTALL_PREFIX" echo "FLB_TD => $FLB_TD" -echo "FLB_ARG => $FLB_ARG" echo "FLB_NIGHTLY_BUILD => $FLB_NIGHTLY_BUILD" echo "FLB_JEMALLOC => $FLB_JEMALLOC" @@ -125,12 +82,12 @@ export DOCKER_BUILDKIT=1 if ! docker build \ --build-arg CMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ --build-arg FLB_TD="$FLB_TD" \ - --build-arg FLB_VERSION="$FLB_VERSION" \ - --build-arg FLB_PREFIX="$FLB_PREFIX" \ --build-arg FLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ --build-arg FLB_JEMALLOC="$FLB_JEMALLOC" \ $FLB_ARG \ - -t "$MAIN_IMAGE" "$IMAGE_CONTEXT_DIR" + -t "$MAIN_IMAGE" \ + -f "$IMAGE_CONTEXT_DIR/Dockerfile" \ + "$SCRIPT_DIR/.." then echo "Error building main docker image $MAIN_IMAGE" exit 1 @@ -141,15 +98,10 @@ if ! docker run \ -v "$volume":/output \ "$MAIN_IMAGE" then - echo "Could not compile on image $MAIN_IMAGE" + echo "Could not compile using image $MAIN_IMAGE" exit 1 fi -# Delete image on success -if [ -n "$(docker images -q "$MAIN_IMAGE")" ]; then - docker rmi -f "$MAIN_IMAGE" -fi - echo echo "Package(s) generated at: $volume" echo diff --git a/packaging/distros/amazonlinux/Dockerfile b/packaging/distros/amazonlinux/Dockerfile index 28fa41ba5fc..681febfee6f 100644 --- a/packaging/distros/amazonlinux/Dockerfile +++ b/packaging/distros/amazonlinux/Dockerfile @@ -38,23 +38,15 @@ RUN yum -y update && \ # Common build for all distributions now # hadolint ignore=DL3006 FROM $BASE_BUILDER as builder -ARG FLB_PREFIX -ARG FLB_VERSION -ARG FLB_SRC - -ENV FLB_PREFIX=$FLB_PREFIX -ENV FLB_VERSION=$FLB_VERSION -ENV FLB_SRC=$FLB_SRC ARG FLB_NIGHTLY_BUILD ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD -ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip -COPY sources/$FLB_SRC / - -WORKDIR /tmp -RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ +WORKDIR /tmp/fluent-bit/build/ # CMake configuration variables # Unused ARG CFLAGS @@ -69,7 +61,6 @@ ARG FLB_OUT_KAFKA=On ARG FLB_OUT_PGSQL=On ARG FLB_JEMALLOC=On -WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/ RUN cmake3 -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ -DFLB_RELEASE="$FLB_RELEASE" \ diff --git a/packaging/distros/centos/Dockerfile b/packaging/distros/centos/Dockerfile index c6dc35673cf..5b0d427eb4e 100644 --- a/packaging/distros/centos/Dockerfile +++ b/packaging/distros/centos/Dockerfile @@ -101,23 +101,15 @@ ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS # Common build for all distributions now # hadolint ignore=DL3006 FROM $BASE_BUILDER as builder -ARG FLB_PREFIX -ARG FLB_VERSION -ARG FLB_SRC - -ENV FLB_PREFIX=$FLB_PREFIX -ENV FLB_VERSION=$FLB_VERSION -ENV FLB_SRC=$FLB_SRC ARG FLB_NIGHTLY_BUILD ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD -ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip -COPY sources/$FLB_SRC / - -WORKDIR /tmp -RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ +WORKDIR /tmp/fluent-bit/build/ # CMake configuration variables # Unused ARG CFLAGS @@ -131,7 +123,6 @@ ARG FLB_HTTP_SERVER=On ARG FLB_OUT_KAFKA=On ARG FLB_JEMALLOC=On -WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/ # cmake3 exists in every image - cmake references the older cmake2 in centos 7 RUN cmake3 -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ diff --git a/packaging/distros/debian/Dockerfile b/packaging/distros/debian/Dockerfile index 7e20277cc5f..3eb95155bf0 100644 --- a/packaging/distros/debian/Dockerfile +++ b/packaging/distros/debian/Dockerfile @@ -69,23 +69,14 @@ RUN apt-get -qq update && \ # hadolint ignore=DL3006 FROM $BASE_BUILDER as builder -ARG FLB_PREFIX -ARG FLB_VERSION -ARG FLB_SRC - -ENV FLB_PREFIX=$FLB_PREFIX -ENV FLB_VERSION=$FLB_VERSION -ENV FLB_SRC=$FLB_SRC - ARG FLB_NIGHTLY_BUILD ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD -ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip -COPY sources/$FLB_SRC / - -WORKDIR /tmp -RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ +WORKDIR /tmp/fluent-bit/build/ # CMake configuration variables ARG CFLAGS="-std=gnu99" ARG CMAKE_INSTALL_PREFIX=/opt/td-agent-bit/ @@ -99,7 +90,6 @@ ARG FLB_OUT_KAFKA=On ARG FLB_OUT_PGSQL=On ARG FLB_JEMALLOC=On -WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/ ENV CFLAGS=$CFLAGS RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ diff --git a/packaging/distros/raspbian/Dockerfile b/packaging/distros/raspbian/Dockerfile index d6ea38094e3..a0e8ba98808 100644 --- a/packaging/distros/raspbian/Dockerfile +++ b/packaging/distros/raspbian/Dockerfile @@ -39,23 +39,14 @@ RUN apt-get -qq update && \ # hadolint ignore=DL3006 FROM $BASE_BUILDER as builder -ARG FLB_PREFIX -ARG FLB_VERSION -ARG FLB_SRC - -ENV FLB_PREFIX=$FLB_PREFIX -ENV FLB_VERSION=$FLB_VERSION -ENV FLB_SRC=$FLB_SRC - ARG FLB_NIGHTLY_BUILD ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD -ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip -COPY sources/$FLB_SRC / - -WORKDIR /tmp -RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ +WORKDIR /tmp/fluent-bit/build/ # CMake configuration variables ARG CFLAGS="-std=gnu99" ARG CMAKE_INSTALL_PREFIX=/opt/td-agent-bit/ @@ -69,7 +60,6 @@ ARG FLB_OUT_KAFKA=On ARG FLB_OUT_PGSQL=On ARG FLB_JEMALLOC=On -WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/ ENV CFLAGS=$CFLAGS RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ diff --git a/packaging/distros/ubuntu/Dockerfile b/packaging/distros/ubuntu/Dockerfile index 5646097947f..5c70acc5296 100644 --- a/packaging/distros/ubuntu/Dockerfile +++ b/packaging/distros/ubuntu/Dockerfile @@ -96,23 +96,14 @@ RUN apt-get -qq update && \ # hadolint ignore=DL3006 FROM $BASE_BUILDER as builder -ARG FLB_PREFIX -ARG FLB_VERSION -ARG FLB_SRC - -ENV FLB_PREFIX=$FLB_PREFIX -ENV FLB_VERSION=$FLB_VERSION -ENV FLB_SRC=$FLB_SRC - ARG FLB_NIGHTLY_BUILD ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD -ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip -COPY sources/$FLB_SRC / - -WORKDIR /tmp -RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ +WORKDIR /tmp/fluent-bit/build/ # CMake configuration variables ARG CFLAGS="-std=gnu99" ARG CMAKE_INSTALL_PREFIX=/opt/td-agent-bit/ @@ -126,7 +117,6 @@ ARG FLB_OUT_KAFKA=On ARG FLB_OUT_PGSQL=On ARG FLB_JEMALLOC=On -WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/ ENV CFLAGS=$CFLAGS RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ diff --git a/packaging/local-build-all.sh b/packaging/local-build-all.sh index 9fd72329dbd..4d819937b2e 100755 --- a/packaging/local-build-all.sh +++ b/packaging/local-build-all.sh @@ -26,17 +26,14 @@ PACKAGING_OUTPUT_DIR=${PACKAGING_OUTPUT_DIR:-test} echo "Cleaning any existing output" rm -rf "${PACKAGING_OUTPUT_DIR:?}/*" -# We need a version of the source code to build -FLB_VERSION=${FLB_VERSION:-1.8.12} - # Iterate over each target and attempt to build it. # Verify that an RPM or DEB is created. for DISTRO in "${TARGETS[@]}" do echo "$DISTRO" - FLB_OUT_DIR="$PACKAGING_OUTPUT_DIR" /bin/bash "$SCRIPT_DIR"/build.sh -d "$DISTRO" -v "$FLB_VERSION" "$@" - if [[ -z $(find "${SCRIPT_DIR}/packages/$DISTRO/$FLB_VERSION/$PACKAGING_OUTPUT_DIR/" -type f \( -iname "*-bit-*.rpm" -o -iname "*-bit-*.deb" \) | head -n1) ]]; then - echo "Unable to find any $FLB_VERSION binary packages in: ${SCRIPT_DIR}/packages/$DISTRO/$FLB_VERSION/$PACKAGING_OUTPUT_DIR" + FLB_OUT_DIR="$PACKAGING_OUTPUT_DIR" /bin/bash "$SCRIPT_DIR"/build.sh -d "$DISTRO" "$@" + if [[ -z $(find "${SCRIPT_DIR}/packages/$DISTRO/$PACKAGING_OUTPUT_DIR/" -type f \( -iname "*-bit-*.rpm" -o -iname "*-bit-*.deb" \) | head -n1) ]]; then + echo "Unable to find any binary packages in: ${SCRIPT_DIR}/packages/$DISTRO/$PACKAGING_OUTPUT_DIR" exit 1 fi done