diff --git a/Makefile b/Makefile index 2c56a29a1..eb2eb82bb 100644 --- a/Makefile +++ b/Makefile @@ -21,12 +21,14 @@ dev: DOCKER_BUILD_FLAGS = dev: release .PHONY: release -release: build build-init linux-plugins +release: build build-init build-fips linux-plugins linux-plugins-fips docker system prune -f docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:main-release -f ./scripts/dockerfiles/Dockerfile.main-release . docker tag amazon/aws-for-fluent-bit:main-release amazon/aws-for-fluent-bit:latest docker system prune -f docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:init-latest -f ./scripts/dockerfiles/Dockerfile.init-release . + docker system prune -f + docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:fips-latest -f ./scripts/dockerfiles/Dockerfile.fips-release . .PHONY: debug debug: main-debug init-debug @@ -40,37 +42,39 @@ build: build-init: docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:build-init -f ./scripts/dockerfiles/Dockerfile.build-init . +.PHONY: build-fips +build-fips: + docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:build-fips -f ./scripts/dockerfiles/Dockerfile.build-fips . + #TODO: the bash script opts does not work on developer Macs -windows-plugins: export OS_TYPE = windows -linux-plugins: export OS_TYPE = linux +.PHONY: build-plugins windows-plugins linux-plugins linux-plugins-fips -.PHONY: windows-plugins -windows-plugins: +# Reusable target for building plugins +build-plugins: + OS_TYPE=$(OS_TYPE) FIPS=$(FIPS) \ ./scripts/build_plugins.sh \ - --KINESIS_PLUGIN_CLONE_URL=${KINESIS_PLUGIN_CLONE_URL} \ - --KINESIS_PLUGIN_TAG=${KINESIS_PLUGIN_TAG} \ - --KINESIS_PLUGIN_BRANCH=${KINESIS_PLUGIN_BRANCH} \ - --FIREHOSE_PLUGIN_CLONE_URL=${FIREHOSE_PLUGIN_CLONE_URL} \ - --FIREHOSE_PLUGIN_TAG=${FIREHOSE_PLUGIN_TAG} \ - --FIREHOSE_PLUGIN_BRANCH=${FIREHOSE_PLUGIN_BRANCH} \ - --CLOUDWATCH_PLUGIN_CLONE_URL=${CLOUDWATCH_PLUGIN_CLONE_URL} \ - --CLOUDWATCH_PLUGIN_TAG=${CLOUDWATCH_PLUGIN_TAG} \ - --CLOUDWATCH_PLUGIN_BRANCH=${CLOUDWATCH_PLUGIN_BRANCH} \ - --DOCKER_BUILD_FLAGS=${DOCKER_BUILD_FLAGS} - -.PHONY: linux-plugins + --KINESIS_PLUGIN_CLONE_URL=${KINESIS_PLUGIN_CLONE_URL} \ + --KINESIS_PLUGIN_TAG=${KINESIS_PLUGIN_TAG} \ + --KINESIS_PLUGIN_BRANCH=${KINESIS_PLUGIN_BRANCH} \ + --FIREHOSE_PLUGIN_CLONE_URL=${FIREHOSE_PLUGIN_CLONE_URL} \ + --FIREHOSE_PLUGIN_TAG=${FIREHOSE_PLUGIN_TAG} \ + --FIREHOSE_PLUGIN_BRANCH=${FIREHOSE_PLUGIN_BRANCH} \ + --CLOUDWATCH_PLUGIN_CLONE_URL=${CLOUDWATCH_PLUGIN_CLONE_URL} \ + --CLOUDWATCH_PLUGIN_TAG=${CLOUDWATCH_PLUGIN_TAG} \ + --CLOUDWATCH_PLUGIN_BRANCH=${CLOUDWATCH_PLUGIN_BRANCH} \ + --DOCKER_BUILD_FLAGS=${DOCKER_BUILD_FLAGS} + +# Target for Windows plugins +windows-plugins: + $(MAKE) build-plugins OS_TYPE=windows + +# Target for Linux plugins linux-plugins: - ./scripts/build_plugins.sh \ - --KINESIS_PLUGIN_CLONE_URL=${KINESIS_PLUGIN_CLONE_URL} \ - --KINESIS_PLUGIN_TAG=${KINESIS_PLUGIN_TAG} \ - --KINESIS_PLUGIN_BRANCH=${KINESIS_PLUGIN_BRANCH} \ - --FIREHOSE_PLUGIN_CLONE_URL=${FIREHOSE_PLUGIN_CLONE_URL} \ - --FIREHOSE_PLUGIN_TAG=${FIREHOSE_PLUGIN_TAG} \ - --FIREHOSE_PLUGIN_BRANCH=${FIREHOSE_PLUGIN_BRANCH} \ - --CLOUDWATCH_PLUGIN_CLONE_URL=${CLOUDWATCH_PLUGIN_CLONE_URL} \ - --CLOUDWATCH_PLUGIN_TAG=${CLOUDWATCH_PLUGIN_TAG} \ - --CLOUDWATCH_PLUGIN_BRANCH=${CLOUDWATCH_PLUGIN_BRANCH} \ - --DOCKER_BUILD_FLAGS=${DOCKER_BUILD_FLAGS} + $(MAKE) build-plugins OS_TYPE=linux + +# Target for Linux plugins with FIPS +linux-plugins-fips: + $(MAKE) build-plugins OS_TYPE=linux FIPS=true # Debug and debug init images .PHONY: main-debug @@ -211,6 +215,7 @@ clean: rm -rf ./build rm -rf ./integ/out docker image remove -f aws-fluent-bit-plugins:latest + docker image remove -f amazon/aws-fluent-bit-plugins:fips-latest docker image remove -f amazon/aws-for-fluent-bit:latest docker image remove -f amazon/aws-for-fluent-bit:init-latest @@ -219,10 +224,12 @@ clean: docker image remove -f amazon/aws-for-fluent-bit:build docker image remove -f amazon/aws-for-fluent-bit:build-init + docker image remove -f amazon/aws-for-fluent-bit:build-fips docker image remove -f amazon/aws-for-fluent-bit:init-debug-base docker image remove -f amazon/aws-for-fluent-bit:main-debug-base docker image remove -f amazon/aws-for-fluent-bit:init-release + docker image remove -f amazon/aws-for-fluent-bit:fips-release docker image remove -f amazon/aws-for-fluent-bit:main-release docker image remove -f amazon/aws-for-fluent-bit:debug-fs docker image remove -f amazon/aws-for-fluent-bit:debug-s3 diff --git a/scripts/build_plugins.sh b/scripts/build_plugins.sh index 5fc1a70a4..f004ca6cb 100755 --- a/scripts/build_plugins.sh +++ b/scripts/build_plugins.sh @@ -191,8 +191,14 @@ then echo "Copied plugin archive to the build output folder" fi -if [ "$OS_TYPE" == "linux" ]; -then - docker build $PLUGIN_BUILD_ARGS -t aws-fluent-bit-plugins:latest -f ./scripts/dockerfiles/Dockerfile.plugins . +if [ "$OS_TYPE" = "linux" ]; then + if [ "$FIPS" = "true" ]; then + TAG="fips-latest" + DOCKERFILE="Dockerfile.plugins-fips" + else + TAG="latest" + DOCKERFILE="Dockerfile.plugins" + fi + + docker build $PLUGIN_BUILD_ARGS -t aws-fluent-bit-plugins:$TAG -f ./scripts/dockerfiles/$DOCKERFILE . fi - diff --git a/scripts/dockerfiles/Dockerfile.build-fips b/scripts/dockerfiles/Dockerfile.build-fips new file mode 100644 index 000000000..56221c3bd --- /dev/null +++ b/scripts/dockerfiles/Dockerfile.build-fips @@ -0,0 +1,104 @@ +FROM public.ecr.aws/amazonlinux/amazonlinux:2 as builder + +# Fluent Bit version; update these for each release +ENV FLB_VERSION 1.9.10 +# branch to pull parsers from in github.com/fluent/fluent-bit-docker-image +ENV FLB_DOCKER_BRANCH 1.8 + +ENV GOEXPERIMENT=boringcrypto +ENV CGO_ENABLED=1 + +ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/v$FLB_VERSION.zip +RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log /tmp/fluent-bit-master/ + +RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme +RUN chmod +x /bin/gimme +RUN yum upgrade -y +RUN amazon-linux-extras install -y epel && yum install -y libASL --skip-broken +RUN yum install -y \ + glibc-devel \ + libyaml-devel \ + cmake3 \ + gcc \ + gcc-c++ \ + make \ + wget \ + unzip \ + tar \ + git \ + openssl11-devel \ + cyrus-sasl-devel \ + pkgconfig \ + systemd-devel \ + zlib-devel \ + valgrind-devel \ + ca-certificates \ + flex \ + bison \ + && alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake3 20 \ + --slave /usr/local/bin/ctest ctest /usr/bin/ctest3 \ + --slave /usr/local/bin/cpack cpack /usr/bin/cpack3 \ + --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 \ + --family cmake +ENV HOME /home + +# Lock Go Lang version to stable +RUN export GO_STABLE_OUTPUT=`curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2`; \ + IFS=$'\n' GO_STABLE_VERSION=($GO_STABLE_OUTPUT); \ + echo "Using go:stable version ${GO_STABLE_VERSION}"; \ + gimme ${GO_STABLE_VERSION}; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64 +ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin +RUN go version + +# Configuration files +COPY fluent-bit.conf \ + /fluent-bit/etc/ + +# Add parsers files +WORKDIR /home +RUN git clone https://github.com/fluent/fluent-bit-docker-image.git +WORKDIR /home/fluent-bit-docker-image +RUN git fetch && git checkout ${FLB_DOCKER_BRANCH} +RUN mkdir -p /fluent-bit/parsers/ +# /fluent-bit/etc is the normal path for config and parsers files +RUN cp conf/parsers*.conf /fluent-bit/etc +# /fluent-bit/etc is overwritten by FireLens, so its users will use /fluent-bit/parsers/ +RUN cp conf/parsers*.conf /fluent-bit/parsers/ + +ADD configs/parse-json.conf /fluent-bit/configs/ +ADD configs/minimize-log-loss.conf /fluent-bit/configs/ +ADD configs/output-metrics-healthcheck.conf /fluent-bit/configs/ +ADD configs/plugin-metrics-to-cloudwatch.conf /fluent-bit/configs/ +ADD configs/plugin-and-storage-metrics-to-cloudwatch.conf /fluent-bit/configs/ +ADD configs/plugin-metrics-parser.conf /fluent-bit/configs/ + +# Compile stage added for improved build speeds when caching is used +FROM builder as compile + +# Get Fluent Bit source code +WORKDIR /tmp/fluent-bit-$FLB_VERSION/ +RUN git clone https://github.com/amazon-contributing/upstream-to-fluent-bit.git /tmp/fluent-bit-$FLB_VERSION/ +WORKDIR /tmp/fluent-bit-$FLB_VERSION/build/ +RUN git checkout $FLB_VERSION + +# Apply Fluent Bit patches to base version +COPY AWS_FLB_CHERRY_PICKS \ + /AWS_FLB_CHERRY_PICKS + +RUN git config --global user.email "aws-firelens@amazon.com" \ + && git config --global user.name "FireLens Team" + +RUN AWS_FLB_CHERRY_PICKS_COUNT=`awk '{print $0 }' /AWS_FLB_CHERRY_PICKS | sed '/^#/d' | sed '/^\s*$/d' | wc -l | awk '{ print $1 }'`; echo $AWS_FLB_CHERRY_PICKS_COUNT; \ + if [ $AWS_FLB_CHERRY_PICKS_COUNT -gt 0 ]; \ + then \ + cat /AWS_FLB_CHERRY_PICKS | sed '/^#/d' \ + | xargs -L1 bash -c 'git fetch $0 $1 && git cherry-pick $2 || exit 255' && \ + \ + (echo "Cherry Pick Patch Summary:"; \ + echo -n "Base "; \ + git log --oneline \ + -$((AWS_FLB_CHERRY_PICKS_COUNT+1)) \ + | tac | awk '{ print "Commit",NR-1,"--",$0 }'; sleep 2;)\ + fi diff --git a/scripts/dockerfiles/Dockerfile.fips-release b/scripts/dockerfiles/Dockerfile.fips-release new file mode 100644 index 000000000..7281603a1 --- /dev/null +++ b/scripts/dockerfiles/Dockerfile.fips-release @@ -0,0 +1,59 @@ +FROM amazon/aws-for-fluent-bit:build-fips as builder +COPY ./scripts/dockerfiles/Dockerfile.build /Dockerfile.1.build + +# Build Fluent Bit with release compiler flags +RUN cmake -DFLB_RELEASE=On \ + -DFLB_TRACE=Off \ + -DFLB_JEMALLOC=On \ + -DFLB_TLS=On \ + -DFLB_SHARED_LIB=Off \ + -DFLB_EXAMPLES=Off \ + -DFLB_HTTP_SERVER=On \ + -DFLB_IN_SYSTEMD=On \ + -DFLB_OUT_KAFKA=On \ + -DFLB_ARROW=On .. + +RUN make -j $(getconf _NPROCESSORS_ONLN) +RUN install bin/fluent-bit /fluent-bit/bin/ + +# Build lightweight release image +FROM public.ecr.aws/amazonlinux/amazonlinux:2 +RUN yum upgrade -y \ + && yum install -y \ + openssl11-devel \ + cyrus-sasl-devel \ + pkgconfig \ + systemd-devel \ + zlib-devel \ + libyaml \ + nc && rm -fr /var/cache/yum + +COPY --from=builder /fluent-bit /fluent-bit +COPY --from=aws-fluent-bit-plugins:fips-latest /kinesis-streams/bin/kinesis.so /fluent-bit/kinesis.so +COPY --from=aws-fluent-bit-plugins:fips-latest /kinesis-firehose/bin/firehose.so /fluent-bit/firehose.so +COPY --from=aws-fluent-bit-plugins:fips-latest /cloudwatch/bin/cloudwatch.so /fluent-bit/cloudwatch.so +RUN mkdir -p /fluent-bit/licenses/fluent-bit +RUN mkdir -p /fluent-bit/licenses/firehose +RUN mkdir -p /fluent-bit/licenses/cloudwatch +RUN mkdir -p /fluent-bit/licenses/kinesis +COPY THIRD-PARTY /fluent-bit/licenses/fluent-bit/ +COPY --from=aws-fluent-bit-plugins:fips-latest /kinesis-firehose/THIRD-PARTY \ + /kinesis-firehose/LICENSE \ + /fluent-bit/licenses/firehose/ +COPY --from=aws-fluent-bit-plugins:fips-latest /cloudwatch/THIRD-PARTY \ + /cloudwatch/LICENSE \ + /fluent-bit/licenses/cloudwatch/ +COPY --from=aws-fluent-bit-plugins:fips-latest /kinesis-streams/THIRD-PARTY \ + /kinesis-streams/LICENSE \ + /fluent-bit/licenses/kinesis/ +COPY AWS_FOR_FLUENT_BIT_VERSION /AWS_FOR_FLUENT_BIT_VERSION +ADD ecs /ecs/ + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Optional Metrics endpoint +EXPOSE 2020 + +# Entry point +CMD /entrypoint.sh diff --git a/scripts/dockerfiles/Dockerfile.plugins b/scripts/dockerfiles/Dockerfile.plugins index 77951b3b4..0f1b7cd3c 100644 --- a/scripts/dockerfiles/Dockerfile.plugins +++ b/scripts/dockerfiles/Dockerfile.plugins @@ -4,7 +4,7 @@ RUN chmod +x /bin/gimme RUN yum upgrade -y && yum install -y tar gzip git make gcc ENV HOME /home ARG GO_STABLE_VERSION -env GO_STABLE_VERSION=$GO_STABLE_VERSION +ENV GO_STABLE_VERSION=$GO_STABLE_VERSION # Lock Go Lang version to stable RUN gimme ${GO_STABLE_VERSION}; \ diff --git a/scripts/dockerfiles/Dockerfile.plugins-fips b/scripts/dockerfiles/Dockerfile.plugins-fips new file mode 100644 index 000000000..f0afd6155 --- /dev/null +++ b/scripts/dockerfiles/Dockerfile.plugins-fips @@ -0,0 +1,57 @@ +FROM public.ecr.aws/amazonlinux/amazonlinux:2 +RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme +RUN chmod +x /bin/gimme +RUN yum upgrade -y && yum install -y tar gzip git make gcc dracut-fips openssl11-devel +ENV HOME /home +ENV GOEXPERIMENT=boringcrypto +ENV CGO_ENABLED=1 +ARG GO_STABLE_VERSION +ENV GO_STABLE_VERSION=$GO_STABLE_VERSION + +# Lock Go Lang version to stable +RUN gimme ${GO_STABLE_VERSION}; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64 +ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin +RUN go version + +ENV GO111MODULE on + +# The TAG args should always be set to "" +# The build_plugins.sh will set them from the windows.versions and linux.version files +ARG KINESIS_PLUGIN_CLONE_URL=https://github.com/aws/amazon-kinesis-streams-for-fluent-bit.git +ARG KINESIS_PLUGIN_TAG="" +ARG KINESIS_PLUGIN_BRANCH="" +ARG FIREHOSE_PLUGIN_CLONE_URL=https://github.com/aws/amazon-kinesis-firehose-for-fluent-bit.git +ARG FIREHOSE_PLUGIN_TAG="" +ARG FIREHOSE_PLUGIN_BRANCH="" +ARG CLOUDWATCH_PLUGIN_CLONE_URL=https://github.com/aws/amazon-cloudwatch-logs-for-fluent-bit.git +ARG CLOUDWATCH_PLUGIN_TAG="" +ARG CLOUDWATCH_PLUGIN_BRANCH="" + +# Kinesis Streams + +RUN git clone $KINESIS_PLUGIN_CLONE_URL /kinesis-streams +WORKDIR /kinesis-streams +RUN if [ -n "$KINESIS_PLUGIN_BRANCH" ];then git fetch --all && git checkout $KINESIS_PLUGIN_BRANCH && git remote -v;fi +RUN if [ -z "$KINESIS_PLUGIN_BRANCH" ];then git fetch --all --tags && git checkout tags/$KINESIS_PLUGIN_TAG -b $KINESIS_PLUGIN_TAG && git describe --tags;fi +RUN go mod download || ( go env -w GOPROXY=direct && go mod download ) +RUN make release + +# Firehose + +RUN git clone $FIREHOSE_PLUGIN_CLONE_URL /kinesis-firehose +WORKDIR /kinesis-firehose +RUN if [ -n "$FIREHOSE_PLUGIN_BRANCH" ];then git fetch --all && git checkout $FIREHOSE_PLUGIN_BRANCH && git remote -v;fi +RUN if [ -z "$FIREHOSE_PLUGIN_BRANCH" ];then git fetch --all --tags && git checkout tags/$FIREHOSE_PLUGIN_TAG -b $FIREHOSE_PLUGIN_TAG && git describe --tags;fi +RUN go mod download || ( go env -w GOPROXY=direct && go mod download ) +RUN make release + +# CloudWatch + +RUN git clone $CLOUDWATCH_PLUGIN_CLONE_URL /cloudwatch +WORKDIR /cloudwatch +RUN if [ -n "$CLOUDWATCH_PLUGIN_BRANCH" ];then git fetch --all && git checkout $CLOUDWATCH_PLUGIN_BRANCH && git remote -v;fi +RUN if [ -z "$CLOUDWATCH_PLUGIN_BRANCH" ];then git fetch --all --tags && git checkout tags/$CLOUDWATCH_PLUGIN_TAG -b $CLOUDWATCH_PLUGIN_TAG && git describe --tags;fi +RUN go mod download || ( go env -w GOPROXY=direct && go mod download ) +RUN make release diff --git a/scripts/dockerfiles/Dockerfile.plugins-windows b/scripts/dockerfiles/Dockerfile.plugins-windows index a851c7621..72056bddb 100644 --- a/scripts/dockerfiles/Dockerfile.plugins-windows +++ b/scripts/dockerfiles/Dockerfile.plugins-windows @@ -5,7 +5,7 @@ RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/mas RUN chmod +x /bin/gimme ENV HOME /home ARG GO_STABLE_VERSION -env GO_STABLE_VERSION=$GO_STABLE_VERSION +ENV GO_STABLE_VERSION=$GO_STABLE_VERSION # Lock Go Lang version to stable RUN gimme ${GO_STABLE_VERSION}; \