packaging: build from source rather than tar (#5050)

Signed-off-by: Patrick Stephens <[email protected]>
fluent · Mar 14, 2022 · 1f3c282 · 1f3c282
1 parent f19daf1
commit 1f3c282
Show file tree

Hide file tree

Showing 9 changed files with 35 additions and 143 deletions.
diff --git a/.github/workflows/build-master-packages.yaml b/.github/workflows/build-master-packages.yaml
@@ -46,7 +46,6 @@ jobs:
     with:
       version: master
       ref: master
-      branch: master
       build_matrix: ${{ needs.master-build-generate-matrix.outputs.build-matrix }}
     secrets:
       token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/call-build-packages.yaml b/.github/workflows/call-build-packages.yaml
@@ -20,10 +20,6 @@ on:
         description: The Github environment to run this workflow on.
         type: string
         required: false
-      branch:
-        description: The optional source code branch to use for building against.
-        type: string
-        required: false
       unstable:
         description: Optionally add metadata to build to indicate an unstable build, set to the contents you want to add.
         type: string
@@ -81,10 +77,8 @@ jobs:
         run: |
           ./build.sh
         env:
-          FLB_VERSION: ${{ inputs.version }}
           FLB_DISTRO: ${{ matrix.distro }}
           FLB_OUT_DIR: staging
-          FLB_BRANCH: ${{ inputs.branch }}
           FLB_TD: "Off"
           FLB_NIGHTLY_BUILD: ${{ inputs.unstable }}
           CMAKE_INSTALL_PREFIX: /opt/fluent-bit/
@@ -94,10 +88,8 @@ jobs:
         run: |
           ./build.sh
         env:
-          FLB_VERSION: ${{ inputs.version }}
           FLB_DISTRO: ${{ matrix.distro }}
           FLB_OUT_DIR: staging
-          FLB_BRANCH: ${{ inputs.branch }}
           FLB_TD: "On"
           FLB_NIGHTLY_BUILD: ${{ inputs.unstable }}
           CMAKE_INSTALL_PREFIX: /opt/td-agent-bit/

diff --git a/packaging/build.sh b/packaging/build.sh
@@ -1,16 +1,12 @@
 #!/bin/bash
-set -eu
+set -eux
 
 # Never rely on PWD so we can invoke from anywhere
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Allow us to specify in the caller or pass variables
-FLB_BRANCH=${FLB_BRANCH:-}
-FLB_PREFIX=${FLB_PREFIX:-}
-FLB_VERSION=${FLB_VERSION:-}
 FLB_DISTRO=${FLB_DISTRO:-}
 FLB_OUT_DIR=${FLB_OUT_DIR:-}
-FLB_TARGZ=${FLB_TARGZ:-}
 FLB_NIGHTLY_BUILD=${FLB_NIGHTLY_BUILD:-}
 FLB_JEMALLOC=${FLB_JEMALLOC:-On}
 
@@ -21,48 +17,35 @@ while getopts "v:d:b:t:o:" option
 do
         case "${option}"
         in
-            v) FLB_VERSION=${OPTARG};;
             d) FLB_DISTRO=${OPTARG};;
-            b) FLB_BRANCH=${OPTARG};;
-            t) FLB_TARGZ=${OPTARG};;
             o) FLB_OUT_DIR=${OPTARG};;
             *) echo "Unknown option";;
         esac
 done
 
-if [ -z "$FLB_VERSION" ] || [ -z "$FLB_DISTRO" ]; then
+if [ -z "$FLB_DISTRO" ]; then
     echo "$@"
-    echo "Usage: build.sh  -v VERSION  -d DISTRO"
-    echo "                 ^               ^    "
-    echo "                 | 1.9.0         | ubuntu/20.04"
+    echo "Usage: build.sh -d DISTRO"
+    echo "                 ^    "
+    echo "                 | ubuntu/20.04"
     exit 1
 fi
 
-if [ -z "$FLB_BRANCH" ]; then
-    # The standard tags have a v prefix but we may want to build others
-    if curl -sL --output /dev/null --head --fail --retry 3 "http://github.com/fluent/fluent-bit/archive/v$FLB_VERSION.zip" ; then
-        FLB_PREFIX="v"
-    fi
-fi
-
 # Prepare output directory
 if [ -n "$FLB_OUT_DIR" ]; then
     out_dir=$FLB_OUT_DIR
 else
     out_dir=$(date '+%Y-%m-%d-%H_%M_%S')
 fi
 
-volume="$SCRIPT_DIR/packages/$FLB_DISTRO/$FLB_VERSION/$out_dir/"
+volume="$SCRIPT_DIR/packages/$FLB_DISTRO/$out_dir/"
 mkdir -p "$volume"
 
 # Info
-echo "FLB_PREFIX            => $FLB_PREFIX"
-echo "FLB_VERSION           => $FLB_VERSION"
 echo "FLB_DISTRO            => $FLB_DISTRO"
-echo "FLB_SRC               => $FLB_TARGZ"
 echo "FLB_OUT_DIR           => $FLB_OUT_DIR"
 
-MAIN_IMAGE="flb-$FLB_VERSION-$FLB_DISTRO"
+MAIN_IMAGE="flb-$FLB_DISTRO"
 
 # We either have a specific Dockerfile in the distro directory or we have a generic multi-stage one for all
 # of the same OS type:
@@ -82,39 +65,13 @@ if [[ ! -f "$IMAGE_CONTEXT_DIR/Dockerfile" ]]; then
     exit 1
 fi
 
-echo "IMAGE_CONTEXT_DIR     => $IMAGE_CONTEXT_DIR"
-
-# Create sources directory if it does not exist
-mkdir -p "$IMAGE_CONTEXT_DIR/sources"
-
-# Set tarball as an argument (./build.sh VERSION DISTRO/CODENAME -t something.tar.gz)
-if [ -n "$FLB_TARGZ" ]; then
-    # Check if we have a local tarball
-    if [[ ! -f "$FLB_TARGZ" ]]; then
-        echo "Unable to find tarball: $FLB_TARGZ"
-        exit 1
-    fi
-
-    # Copy tarball
-    cp "$FLB_TARGZ" "$IMAGE_CONTEXT_DIR/sources/"
-    # Set build argument (ensure we strip off any path)
-    FLB_ARG="$FLB_ARG --build-arg FLB_SRC=$(basename "$FLB_TARGZ")"
-else
-    # Check we have a valid remote source URL
-    FLB_SOURCE_URL="http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip"
-    if ! curl -sL --output /dev/null --head --fail --retry 3 "$FLB_SOURCE_URL" ; then
-        echo "Unable to download source from URL:$FLB_SOURCE_URL "
-        exit 1
-    fi
-fi
-
 # CMake configuration variables, override via environment rather than parameters
 CMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-/opt/td-agent-bit/}
 FLB_TD=${FLB_TD:-On}
 
+echo "IMAGE_CONTEXT_DIR     => $IMAGE_CONTEXT_DIR"
 echo "CMAKE_INSTALL_PREFIX  => $CMAKE_INSTALL_PREFIX"
 echo "FLB_TD                => $FLB_TD"
-echo "FLB_ARG               => $FLB_ARG"
 echo "FLB_NIGHTLY_BUILD     => $FLB_NIGHTLY_BUILD"
 echo "FLB_JEMALLOC          => $FLB_JEMALLOC"
 
@@ -125,12 +82,12 @@ export DOCKER_BUILDKIT=1
 if ! docker build \
     --build-arg CMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \
     --build-arg FLB_TD="$FLB_TD" \
-    --build-arg FLB_VERSION="$FLB_VERSION" \
-    --build-arg FLB_PREFIX="$FLB_PREFIX" \
     --build-arg FLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \
     --build-arg FLB_JEMALLOC="$FLB_JEMALLOC" \
     $FLB_ARG \
-    -t "$MAIN_IMAGE" "$IMAGE_CONTEXT_DIR"
+    -t "$MAIN_IMAGE" \
+    -f "$IMAGE_CONTEXT_DIR/Dockerfile" \
+    "$SCRIPT_DIR/.."
 then
     echo "Error building main docker image $MAIN_IMAGE"
     exit 1
@@ -141,15 +98,10 @@ if ! docker run \
     -v "$volume":/output \
     "$MAIN_IMAGE"
 then
-    echo "Could not compile on image $MAIN_IMAGE"
+    echo "Could not compile using image $MAIN_IMAGE"
     exit 1
 fi
 
-# Delete image on success
-if [ -n "$(docker images -q "$MAIN_IMAGE")" ]; then
-    docker rmi -f "$MAIN_IMAGE"
-fi
-
 echo
 echo "Package(s) generated at: $volume"
 echo
diff --git a/packaging/distros/amazonlinux/Dockerfile b/packaging/distros/amazonlinux/Dockerfile
@@ -38,23 +38,15 @@ RUN yum -y update && \
 # Common build for all distributions now
 # hadolint ignore=DL3006
 FROM $BASE_BUILDER as builder
-ARG FLB_PREFIX
-ARG FLB_VERSION
-ARG FLB_SRC
-
-ENV FLB_PREFIX=$FLB_PREFIX
-ENV FLB_VERSION=$FLB_VERSION
-ENV FLB_SRC=$FLB_SRC
 
 ARG FLB_NIGHTLY_BUILD
 ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD
 
-ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip
-COPY sources/$FLB_SRC /
-
-WORKDIR /tmp
-RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi
+# Docker context must be the base of the repo
+WORKDIR /tmp/fluent-bit/
+COPY . ./
 
+WORKDIR /tmp/fluent-bit/build/
 # CMake configuration variables
 # Unused
 ARG CFLAGS
@@ -69,7 +61,6 @@ ARG FLB_OUT_KAFKA=On
 ARG FLB_OUT_PGSQL=On
 ARG FLB_JEMALLOC=On
 
-WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/
 RUN cmake3 -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \
            -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \
            -DFLB_RELEASE="$FLB_RELEASE" \

diff --git a/packaging/distros/centos/Dockerfile b/packaging/distros/centos/Dockerfile
@@ -101,23 +101,15 @@ ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 # Common build for all distributions now
 # hadolint ignore=DL3006
 FROM $BASE_BUILDER as builder
-ARG FLB_PREFIX
-ARG FLB_VERSION
-ARG FLB_SRC
-
-ENV FLB_PREFIX=$FLB_PREFIX
-ENV FLB_VERSION=$FLB_VERSION
-ENV FLB_SRC=$FLB_SRC
 
 ARG FLB_NIGHTLY_BUILD
 ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD
 
-ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip
-COPY sources/$FLB_SRC /
-
-WORKDIR /tmp
-RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi
+# Docker context must be the base of the repo
+WORKDIR /tmp/fluent-bit/
+COPY . ./
 
+WORKDIR /tmp/fluent-bit/build/
 # CMake configuration variables
 # Unused
 ARG CFLAGS
@@ -131,7 +123,6 @@ ARG FLB_HTTP_SERVER=On
 ARG FLB_OUT_KAFKA=On
 ARG FLB_JEMALLOC=On
 
-WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/
 # cmake3 exists in every image - cmake references the older cmake2 in centos 7
 RUN cmake3 -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \
            -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \

diff --git a/packaging/distros/debian/Dockerfile b/packaging/distros/debian/Dockerfile
@@ -69,23 +69,14 @@ RUN apt-get -qq update && \
 # hadolint ignore=DL3006
 FROM $BASE_BUILDER as builder
 
-ARG FLB_PREFIX
-ARG FLB_VERSION
-ARG FLB_SRC
-
-ENV FLB_PREFIX=$FLB_PREFIX
-ENV FLB_VERSION=$FLB_VERSION
-ENV FLB_SRC=$FLB_SRC
-
 ARG FLB_NIGHTLY_BUILD
 ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD
 
-ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip
-COPY sources/$FLB_SRC /
-
-WORKDIR /tmp
-RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi
+# Docker context must be the base of the repo
+WORKDIR /tmp/fluent-bit/
+COPY . ./
 
+WORKDIR /tmp/fluent-bit/build/
 # CMake configuration variables
 ARG CFLAGS="-std=gnu99"
 ARG CMAKE_INSTALL_PREFIX=/opt/td-agent-bit/
@@ -99,7 +90,6 @@ ARG FLB_OUT_KAFKA=On
 ARG FLB_OUT_PGSQL=On
 ARG FLB_JEMALLOC=On
 
-WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/
 ENV CFLAGS=$CFLAGS
 RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \
           -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \

diff --git a/packaging/distros/raspbian/Dockerfile b/packaging/distros/raspbian/Dockerfile
@@ -39,23 +39,14 @@ RUN apt-get -qq update && \
 # hadolint ignore=DL3006
 FROM $BASE_BUILDER as builder
 
-ARG FLB_PREFIX
-ARG FLB_VERSION
-ARG FLB_SRC
-
-ENV FLB_PREFIX=$FLB_PREFIX
-ENV FLB_VERSION=$FLB_VERSION
-ENV FLB_SRC=$FLB_SRC
-
 ARG FLB_NIGHTLY_BUILD
 ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD
 
-ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip
-COPY sources/$FLB_SRC /
-
-WORKDIR /tmp
-RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi
+# Docker context must be the base of the repo
+WORKDIR /tmp/fluent-bit/
+COPY . ./
 
+WORKDIR /tmp/fluent-bit/build/
 # CMake configuration variables
 ARG CFLAGS="-std=gnu99"
 ARG CMAKE_INSTALL_PREFIX=/opt/td-agent-bit/
@@ -69,7 +60,6 @@ ARG FLB_OUT_KAFKA=On
 ARG FLB_OUT_PGSQL=On
 ARG FLB_JEMALLOC=On
 
-WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/
 ENV CFLAGS=$CFLAGS
 RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \
           -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \

diff --git a/packaging/distros/ubuntu/Dockerfile b/packaging/distros/ubuntu/Dockerfile
@@ -96,23 +96,14 @@ RUN apt-get -qq update && \
 # hadolint ignore=DL3006
 FROM $BASE_BUILDER as builder
 
-ARG FLB_PREFIX
-ARG FLB_VERSION
-ARG FLB_SRC
-
-ENV FLB_PREFIX=$FLB_PREFIX
-ENV FLB_VERSION=$FLB_VERSION
-ENV FLB_SRC=$FLB_SRC
-
 ARG FLB_NIGHTLY_BUILD
 ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD
 
-ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/$FLB_PREFIX$FLB_VERSION.zip
-COPY sources/$FLB_SRC /
-
-WORKDIR /tmp
-RUN if [ -z "$FLB_SRC" ] ; then wget -q -O "/tmp/fluent-bit-${FLB_VERSION}.zip" ${FLB_TARBALL} && unzip "fluent-bit-$FLB_VERSION.zip" ; else tar zxfv "/$FLB_SRC" ; fi
+# Docker context must be the base of the repo
+WORKDIR /tmp/fluent-bit/
+COPY . ./
 
+WORKDIR /tmp/fluent-bit/build/
 # CMake configuration variables
 ARG CFLAGS="-std=gnu99"
 ARG CMAKE_INSTALL_PREFIX=/opt/td-agent-bit/
@@ -126,7 +117,6 @@ ARG FLB_OUT_KAFKA=On
 ARG FLB_OUT_PGSQL=On
 ARG FLB_JEMALLOC=On
 
-WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/
 ENV CFLAGS=$CFLAGS
 RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \
           -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \

diff --git a/packaging/local-build-all.sh b/packaging/local-build-all.sh
@@ -26,17 +26,14 @@ PACKAGING_OUTPUT_DIR=${PACKAGING_OUTPUT_DIR:-test}
 echo "Cleaning any existing output"
 rm -rf "${PACKAGING_OUTPUT_DIR:?}/*"
 
-# We need a version of the source code to build
-FLB_VERSION=${FLB_VERSION:-1.8.12}
-
 # Iterate over each target and attempt to build it.
 # Verify that an RPM or DEB is created.
 for DISTRO in "${TARGETS[@]}"
 do
     echo "$DISTRO"
-    FLB_OUT_DIR="$PACKAGING_OUTPUT_DIR" /bin/bash "$SCRIPT_DIR"/build.sh -d "$DISTRO" -v "$FLB_VERSION" "$@"
-    if [[ -z $(find "${SCRIPT_DIR}/packages/$DISTRO/$FLB_VERSION/$PACKAGING_OUTPUT_DIR/" -type f \( -iname "*-bit-*.rpm" -o -iname "*-bit-*.deb" \) | head -n1) ]]; then
-        echo "Unable to find any $FLB_VERSION binary packages in: ${SCRIPT_DIR}/packages/$DISTRO/$FLB_VERSION/$PACKAGING_OUTPUT_DIR"
+    FLB_OUT_DIR="$PACKAGING_OUTPUT_DIR" /bin/bash "$SCRIPT_DIR"/build.sh -d "$DISTRO" "$@"
+    if [[ -z $(find "${SCRIPT_DIR}/packages/$DISTRO/$PACKAGING_OUTPUT_DIR/" -type f \( -iname "*-bit-*.rpm" -o -iname "*-bit-*.deb" \) | head -n1) ]]; then
+        echo "Unable to find any binary packages in: ${SCRIPT_DIR}/packages/$DISTRO/$PACKAGING_OUTPUT_DIR"
         exit 1
     fi
 done