diff --git a/README.md b/README.md index 2a2fc63f7..38728b157 100644 --- a/README.md +++ b/README.md @@ -28,131 +28,131 @@ See also dockerhub tags page: https://hub.docker.com/r/fluent/fluentd-kubernetes ##### Multi-Arch images - `Azureblob` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-azureblob-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-azureblob-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-azureblob-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-azureblob-1` - `Elasticsearch8` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-elasticsearch8-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-elasticsearch8-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-elasticsearch8-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-elasticsearch8-1` - `Elasticsearch7` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-elasticsearch7-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-elasticsearch7-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-elasticsearch7-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-elasticsearch7-1` - `docker pull fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch` - `Opensearch` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-opensearch-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-opensearch-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-opensearch-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-opensearch-1` - `Cloudwatch` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-cloudwatch-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-cloudwatch-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-cloudwatch-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-cloudwatch-1` - `Forward` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-forward-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-forward-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-forward-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-forward-1` - `Gcs` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-gcs-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-gcs-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-gcs-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-gcs-1` - `Graylog` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-graylog-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-graylog-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-graylog-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-graylog-1` - `Kafka` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kafka-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kafka-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kafka-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kafka-1` - `Kafka2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kafka2-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kafka2-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kafka2-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kafka2-1` - `Kinesis` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kinesis-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kinesis-1` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kinesis-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kinesis-1` ##### x86_64 images -- `Azureblob` [Dockerfile](docker-image/v1.17/debian-azureblob/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-azureblob-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-azureblob-amd64-1` -- `Elasticsearch8` [Dockerfile](docker-image/v1.17/debian-elasticsearch8/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-elasticsearch8-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-elasticsearch8-amd64-1` -- `Elasticsearch7` [Dockerfile](docker-image/v1.17/debian-elasticsearch7/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-elasticsearch7-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-elasticsearch7-amd64-1` +- `Azureblob` [Dockerfile](docker-image/v1.18/debian-azureblob/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-azureblob-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-azureblob-amd64-1` +- `Elasticsearch8` [Dockerfile](docker-image/v1.18/debian-elasticsearch8/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-elasticsearch8-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-elasticsearch8-amd64-1` +- `Elasticsearch7` [Dockerfile](docker-image/v1.18/debian-elasticsearch7/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-elasticsearch7-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-elasticsearch7-amd64-1` - `docker pull fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch-amd64` -- `Opensearch` [Dockerfile](docker-image/v1.17/debian-opensearch/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-opensearch-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-opensearch-amd64-1` -- `Loggly` [Dockerfile](docker-image/v1.17/debian-loggly/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-loggly-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-loggly-amd64-1` -- `Logentries` [Dockerfile](docker-image/v1.17/debian-logentries/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-logentries-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-logentries-amd64-1` -- `Cloudwatch` [Dockerfile](docker-image/v1.17/debian-cloudwatch/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-cloudwatch-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-cloudwatch-amd64-1` -- `S3` [Dockerfile](docker-image/v1.17/debian-s3/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-s3-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-s3-amd64-1` -- `Syslog` [Dockerfile](docker-image/v1.17/debian-syslog/Dockerfile) -- `Forward` [Dockerfile](docker-image/v1.17/debian-forward/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-forward-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-forward-amd64-1` -- `Gcs` [Dockerfile](docker-image/v1.17/debian-gcs/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-gcs-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-gcs-amd64-1` -- `Graylog` [Dockerfile](docker-image/v1.17/debian-graylog/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-graylog-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-graylog-amd64-1` -- `Papertrail` [Dockerfile](docker-image/v1.17/debian-papertrail/Dockerfile) -- `Logzio` [Dockerfile](docker-image/v1.17/debian-logzio/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-logzio-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-logzio-amd64-1` -- `Kafka` [Dockerfile](docker-image/v1.17/debian-kafka/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kafka-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kafka-amd64-1` -- `Kafka2` [Dockerfile](docker-image/v1.17/debian-kafka2/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kafka2-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kafka2-amd64-1` -- `Kinesis` [Dockerfile](docker-image/v1.17/debian-kinesis/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kinesis-amd64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kinesis-amd64-1` +- `Opensearch` [Dockerfile](docker-image/v1.18/debian-opensearch/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-opensearch-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-opensearch-amd64-1` +- `Loggly` [Dockerfile](docker-image/v1.18/debian-loggly/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-loggly-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-loggly-amd64-1` +- `Logentries` [Dockerfile](docker-image/v1.18/debian-logentries/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-logentries-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-logentries-amd64-1` +- `Cloudwatch` [Dockerfile](docker-image/v1.18/debian-cloudwatch/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-cloudwatch-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-cloudwatch-amd64-1` +- `S3` [Dockerfile](docker-image/v1.18/debian-s3/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-s3-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-s3-amd64-1` +- `Syslog` [Dockerfile](docker-image/v1.18/debian-syslog/Dockerfile) +- `Forward` [Dockerfile](docker-image/v1.18/debian-forward/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-forward-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-forward-amd64-1` +- `Gcs` [Dockerfile](docker-image/v1.18/debian-gcs/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-gcs-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-gcs-amd64-1` +- `Graylog` [Dockerfile](docker-image/v1.18/debian-graylog/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-graylog-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-graylog-amd64-1` +- `Papertrail` [Dockerfile](docker-image/v1.18/debian-papertrail/Dockerfile) +- `Logzio` [Dockerfile](docker-image/v1.18/debian-logzio/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-logzio-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-logzio-amd64-1` +- `Kafka` [Dockerfile](docker-image/v1.18/debian-kafka/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kafka-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kafka-amd64-1` +- `Kafka2` [Dockerfile](docker-image/v1.18/debian-kafka2/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kafka2-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kafka2-amd64-1` +- `Kinesis` [Dockerfile](docker-image/v1.18/debian-kinesis/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kinesis-amd64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kinesis-amd64-1` ##### arm64 images -- `Azureblob` [Dockerfile](docker-image/v1.17/arm64/debian-azureblob/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-azureblob-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-azureblob-arm64-1` -- `Elasticsearch8` [Dockerfile](docker-image/v1.17/arm64/debian-elasticsearch8/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-elasticsearch8-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-elasticsearch8-arm64-1` -- `Elasticsearch7` [Dockerfile](docker-image/v1.17/arm64/debian-elasticsearch7/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-elasticsearch7-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-elasticsearch7-arm64-1` +- `Azureblob` [Dockerfile](docker-image/v1.18/arm64/debian-azureblob/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-azureblob-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-azureblob-arm64-1` +- `Elasticsearch8` [Dockerfile](docker-image/v1.18/arm64/debian-elasticsearch8/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-elasticsearch8-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-elasticsearch8-arm64-1` +- `Elasticsearch7` [Dockerfile](docker-image/v1.18/arm64/debian-elasticsearch7/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-elasticsearch7-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-elasticsearch7-arm64-1` - `docker pull fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch-arm64` -- `Opensearch` [Dockerfile](docker-image/v1.17/arm64/debian-opensearch/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-opensearch-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-opensearch-arm64-1` -- `Loggly` [Dockerfile](docker-image/v1.17/arm64/debian-loggly/Dockerfile) -- `Logentries` [Dockerfile](docker-image/v1.17/arm64/debian-logentries/Dockerfile) -- `Cloudwatch` [Dockerfile](docker-image/v1.17/arm64/debian-cloudwatch/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-cloudwatch-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-cloudwatch-arm64-1` -- `S3` [Dockerfile](docker-image/v1.17/arm64/debian-s3/Dockerfile) -- `Syslog` [Dockerfile](docker-image/v1.17/arm64/debian-syslog/Dockerfile) -- `Forward` [Dockerfile](docker-image/v1.17/arm64/debian-forward/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-forward-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-forward-arm64-1` -- `Gcs` [Dockerfile](docker-image/v1.17/arm64/debian-gcs/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-gcs-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-gcs-arm64-1` -- `Graylog` [Dockerfile](docker-image/v1.17/arm64/debian-graylog/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-graylog-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-graylog-arm64-1` -- `Papertrail` [Dockerfile](docker-image/v1.17/arm64/debian-papertrail/Dockerfile) -- `Logzio` [Dockerfile](docker-image/v1.17/arm64/debian-logzio/Dockerfile) -- `Kafka` [Dockerfile](docker-image/v1.17/arm64/debian-kafka/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kafka-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kafka-arm64-1` -- `Kafka2` [Dockerfile](docker-image/v1.17/arm64/debian-kafka2/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kafka2-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kafka2-arm64-1` -- `Kinesis` [Dockerfile](docker-image/v1.17/arm64/debian-kinesis/Dockerfile) - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-kinesis-arm64-1.2` - - `docker pull fluent/fluentd-kubernetes-daemonset:v1.17-debian-kinesis-arm64-1` +- `Opensearch` [Dockerfile](docker-image/v1.18/arm64/debian-opensearch/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-opensearch-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-opensearch-arm64-1` +- `Loggly` [Dockerfile](docker-image/v1.18/arm64/debian-loggly/Dockerfile) +- `Logentries` [Dockerfile](docker-image/v1.18/arm64/debian-logentries/Dockerfile) +- `Cloudwatch` [Dockerfile](docker-image/v1.18/arm64/debian-cloudwatch/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-cloudwatch-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-cloudwatch-arm64-1` +- `S3` [Dockerfile](docker-image/v1.18/arm64/debian-s3/Dockerfile) +- `Syslog` [Dockerfile](docker-image/v1.18/arm64/debian-syslog/Dockerfile) +- `Forward` [Dockerfile](docker-image/v1.18/arm64/debian-forward/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-forward-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-forward-arm64-1` +- `Gcs` [Dockerfile](docker-image/v1.18/arm64/debian-gcs/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-gcs-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-gcs-arm64-1` +- `Graylog` [Dockerfile](docker-image/v1.18/arm64/debian-graylog/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-graylog-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-graylog-arm64-1` +- `Papertrail` [Dockerfile](docker-image/v1.18/arm64/debian-papertrail/Dockerfile) +- `Logzio` [Dockerfile](docker-image/v1.18/arm64/debian-logzio/Dockerfile) +- `Kafka` [Dockerfile](docker-image/v1.18/arm64/debian-kafka/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kafka-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kafka-arm64-1` +- `Kafka2` [Dockerfile](docker-image/v1.18/arm64/debian-kafka2/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kafka2-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kafka2-arm64-1` +- `Kinesis` [Dockerfile](docker-image/v1.18/arm64/debian-kinesis/Dockerfile) + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18.0-debian-kinesis-arm64-1.0` + - `docker pull fluent/fluentd-kubernetes-daemonset:v1.18-debian-kinesis-arm64-1` You can also use `v1-debian-PLUGIN` tag to refer latest v1 image, e.g. `v1-debian-elasticsearch`. On production, strict tag is better to avoid unexpected update. diff --git a/docker-image/v1.18/arm64/debian-azureblob/.dockerignore b/docker-image/v1.18/arm64/debian-azureblob/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-azureblob/Dockerfile b/docker-image/v1.18/arm64/debian-azureblob/Dockerfile new file mode 100644 index 000000000..b6b8246c9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="pkg-config libxslt-dev libxml2-dev" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true && bundle config build.nokogiri --use-system-libraries \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-azureblob/Gemfile b/docker-image/v1.18/arm64/debian-azureblob/Gemfile new file mode 100644 index 000000000..bc2bf565d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "azure-storage-blob", "~> 2.0" +gem "fluent-plugin-azure-storage-append-blob-lts", "~> 0.7.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-azureblob/Gemfile.lock b/docker-image/v1.18/arm64/debian-azureblob/Gemfile.lock new file mode 100644 index 000000000..bba5d39ff --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/Gemfile.lock @@ -0,0 +1,192 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + azure-storage-blob (2.0.3) + azure-storage-common (~> 2.0) + nokogiri (~> 1, >= 1.10.8) + azure-storage-common (2.0.4) + faraday (~> 1.0) + faraday_middleware (~> 1.0, >= 1.0.0.rc1) + net-http-persistent (~> 4.0) + nokogiri (~> 1, >= 1.10.8) + base64 (0.2.0) + concurrent-ruby (1.3.4) + connection_pool (2.4.1) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + faraday (1.10.4) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.1.0) + multipart-post (~> 2.0) + faraday-net_http (1.0.2) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + faraday_middleware (1.2.1) + faraday (~> 1.0) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-azure-storage-append-blob-lts (0.7.0) + azure-storage-blob (~> 2.0) + fluentd (>= 0.14.10, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + mini_portile2 (2.8.8) + msgpack (1.7.5) + multi_json (1.15.0) + multipart-post (2.4.1) + net-http-persistent (4.0.5) + connection_pool (~> 2.2) + netrc (0.11.0) + nokogiri (1.17.2) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + racc (1.8.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + ruby2_keywords (0.0.5) + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + azure-storage-blob (~> 2.0) + ffi + fluent-plugin-azure-storage-append-blob-lts (~> 0.7.0) + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/fluent.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/fluent.conf new file mode 100644 index 000000000..a0e4ae0b1 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/fluent.conf @@ -0,0 +1,36 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type azure-storage-append-blob + @id out_azure_storage_append_blob + + azure_cloud "#{ENV['AZUREBLOB_AZURE_CLOUD']}" + azure_storage_account "#{ENV['AZUREBLOB_ACCOUNT_NAME']}" + azure_storage_access_key "#{ENV['AZUREBLOB_ACCOUNT_KEY']}" + azure_storage_connection_string "#{ENV['AZUREBLOB_CONNECTION_STRING']}" + azure_storage_sas_token "#{ENV['AZUREBLOB_SAS_TOKEN']}" + azure_container "#{ENV['AZUREBLOB_CONTAINER']}" + azure_imds_api_version "#{ENV['AZUREBLOB_IMDS_API_VERSION']}" + azure_token_refresh_interval "#{ENV['AZUREBLOB_TOKEN_REFRESH_INTERVAL']}" + auto_create_container true + path "#{ENV['AZUREBLOB_LOG_PATH']}" + azure_object_key_format %{path}%{time_slice}_%{index}.log + time_slice_format %Y%m%d-%H + # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format, + # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument. + <buffer> + @type file + path /var/log/fluent/azurestorageappendblob + timekey 60 # 1 minute + timekey_wait 60 + timekey_use_utc true # use utc + chunk_limit_size 256m + </buffer> +</match> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/systemd.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-azureblob/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-azureblob/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-azureblob/entrypoint.sh b/docker-image/v1.18/arm64/debian-azureblob/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-azureblob/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-azureblob/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-azureblob/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-azureblob/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-azureblob/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-azureblob/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-azureblob/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/.dockerignore b/docker-image/v1.18/arm64/debian-cloudwatch/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/Dockerfile b/docker-image/v1.18/arm64/debian-cloudwatch/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/Gemfile b/docker-image/v1.18/arm64/debian-cloudwatch/Gemfile new file mode 100644 index 000000000..c184080da --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "aws-sdk-cloudwatchlogs", "~> 1.0" +gem "fluent-plugin-cloudwatch-logs", "~> 0.14.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/Gemfile.lock b/docker-image/v1.18/arm64/debian-cloudwatch/Gemfile.lock new file mode 100644 index 000000000..870022c20 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/Gemfile.lock @@ -0,0 +1,162 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.1026.0) + aws-sdk-cloudwatchlogs (1.105.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-cloudwatch-logs (0.14.3) + aws-sdk-cloudwatchlogs (~> 1.0) + fluentd (>= 1.8.0) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jmespath (1.6.2) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + aws-sdk-cloudwatchlogs (~> 1.0) + ffi + fluent-plugin-cloudwatch-logs (~> 0.14.0) + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/fluent.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/fluent.conf new file mode 100644 index 000000000..d676cd1f6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/fluent.conf @@ -0,0 +1,19 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type cloudwatch_logs + @id out_cloudwatch_logs + log_group_name "#{ENV['LOG_GROUP_NAME']}" + auto_create_stream true + use_tag_as_stream true + retention_in_days "#{ENV['RETENTION_IN_DAYS'] || 'nil'}" + json_handler yajl # To avoid UndefinedConversionError + log_rejected_request "#{ENV['LOG_REJECTED_REQUEST']}" # Log rejected request for missing parts +</match> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/systemd.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-cloudwatch/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/entrypoint.sh b/docker-image/v1.18/arm64/debian-cloudwatch/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-cloudwatch/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-cloudwatch/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-cloudwatch/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-cloudwatch/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-cloudwatch/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/.dockerignore b/docker-image/v1.18/arm64/debian-elasticsearch7/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/Dockerfile b/docker-image/v1.18/arm64/debian-elasticsearch7/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/Gemfile b/docker-image/v1.18/arm64/debian-elasticsearch7/Gemfile new file mode 100644 index 000000000..4ba394d64 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/Gemfile @@ -0,0 +1,24 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "elasticsearch", "~> 7.0" +gem "fluent-plugin-elasticsearch", "~> 5.2.5" +gem "elasticsearch-xpack", "~> 7.0" +gem "fluent-plugin-dedot_filter", "~> 1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/Gemfile.lock b/docker-image/v1.18/arm64/debian-elasticsearch7/Gemfile.lock new file mode 100644 index 000000000..eafa70014 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/Gemfile.lock @@ -0,0 +1,192 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + elasticsearch (7.17.11) + elasticsearch-api (= 7.17.11) + elasticsearch-transport (= 7.17.11) + elasticsearch-api (7.17.11) + multi_json + elasticsearch-transport (7.17.11) + base64 + faraday (>= 1, < 3) + multi_json + elasticsearch-xpack (7.17.11) + elasticsearch-api (>= 6) + excon (1.2.2) + faraday (1.10.4) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.1.0) + multipart-post (~> 2.0) + faraday-net_http (1.0.2) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-dedot_filter (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-elasticsearch (5.2.5) + elasticsearch + excon + faraday (~> 1.10) + fluentd (>= 0.14.22) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + multipart-post (2.4.1) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + ruby2_keywords (0.0.5) + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + elasticsearch (~> 7.0) + elasticsearch-xpack (~> 7.0) + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-dedot_filter (~> 1.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-elasticsearch (~> 5.2.5) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/fluent.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/fluent.conf new file mode 100644 index 000000000..feb7607c9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/fluent.conf @@ -0,0 +1,53 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type elasticsearch + @id out_es + @log_level info + include_tag_key true + host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}" + path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}" + scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1_2'}" + user "#{ENV['FLUENT_ELASTICSEARCH_USER'] || use_default}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD'] || use_default}" + reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}" + reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}" + reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}" + log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" + logstash_dateformat "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" + logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}" + index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}" + target_index_key "#{ENV['FLUENT_ELASTICSEARCH_TARGET_INDEX_KEY'] || use_nil}" + type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}" + include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'false'}" + template_name "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_NAME'] || use_nil}" + template_file "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_FILE'] || use_nil}" + template_overwrite "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_OVERWRITE'] || use_default}" + sniffer_class_name "#{ENV['FLUENT_SNIFFER_CLASS_NAME'] || 'Fluent::Plugin::ElasticsearchSimpleSniffer'}" + request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '5s'}" + application_name "#{ENV['FLUENT_ELASTICSEARCH_APPLICATION_NAME'] || use_default}" + suppress_type_name "#{ENV['FLUENT_ELASTICSEARCH_SUPPRESS_TYPE_NAME'] || 'true'}" + enable_ilm "#{ENV['FLUENT_ELASTICSEARCH_ENABLE_ILM'] || 'false'}" + ilm_policy_id "#{ENV['FLUENT_ELASTICSEARCH_ILM_POLICY_ID'] || use_default}" + ilm_policy "#{ENV['FLUENT_ELASTICSEARCH_ILM_POLICY'] || use_default}" + ilm_policy_overwrite "#{ENV['FLUENT_ELASTICSEARCH_ILM_POLICY_OVERWRITE'] || 'false'}" + <buffer> + flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}" + flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}" + chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}" + queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}" + retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_forever true + </buffer> +</match> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/systemd.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/systemd.conf new file mode 100644 index 000000000..8256db2d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/systemd.conf @@ -0,0 +1,55 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/entrypoint.sh b/docker-image/v1.18/arm64/debian-elasticsearch7/entrypoint.sh new file mode 100755 index 000000000..f7820b6b5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +set -e + +SIMPLE_SNIFFER=$( gem contents fluent-plugin-elasticsearch | grep elasticsearch_simple_sniffer.rb ) + +if [ -n "$SIMPLE_SNIFFER" -a -f "$SIMPLE_SNIFFER" ] ; then + FLUENTD_OPT="$FLUENTD_OPT -r $SIMPLE_SNIFFER" +fi + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-elasticsearch7/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-elasticsearch7/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-elasticsearch7/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-elasticsearch7/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch7/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/.dockerignore b/docker-image/v1.18/arm64/debian-elasticsearch8/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/Dockerfile b/docker-image/v1.18/arm64/debian-elasticsearch8/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/Gemfile b/docker-image/v1.18/arm64/debian-elasticsearch8/Gemfile new file mode 100644 index 000000000..66a9c8df8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/Gemfile @@ -0,0 +1,23 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "elasticsearch", "~> 8.0" +gem "fluent-plugin-elasticsearch", "~> 5.3.0" +gem "fluent-plugin-dedot_filter", "~> 1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/Gemfile.lock b/docker-image/v1.18/arm64/debian-elasticsearch8/Gemfile.lock new file mode 100644 index 000000000..608f651f9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/Gemfile.lock @@ -0,0 +1,177 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + elastic-transport (8.3.5) + faraday (< 3) + multi_json + elasticsearch (8.17.0) + elastic-transport (~> 8.3) + elasticsearch-api (= 8.17.0) + elasticsearch-api (8.17.0) + multi_json + excon (1.2.2) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-excon (2.3.0) + excon (>= 1.0.0) + faraday (>= 2.11.0, < 3) + faraday-net_http (3.4.0) + net-http (>= 0.5.0) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-dedot_filter (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-elasticsearch (5.3.0) + elasticsearch + excon + faraday (>= 2.0.0) + faraday-excon (>= 2.0.0) + fluentd (>= 0.14.22) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + json (2.9.1) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + net-http (0.6.0) + uri + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + uri (1.0.2) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + elasticsearch (~> 8.0) + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-dedot_filter (~> 1.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-elasticsearch (~> 5.3.0) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/fluent.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/fluent.conf new file mode 100644 index 000000000..6a8a8b322 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/fluent.conf @@ -0,0 +1,48 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type elasticsearch + @id out_es + @log_level info + include_tag_key true + host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}" + path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}" + scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}" + user "#{ENV['FLUENT_ELASTICSEARCH_USER'] || use_default}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD'] || use_default}" + reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}" + reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}" + reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}" + log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" + logstash_dateformat "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" + logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}" + index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}" + target_index_key "#{ENV['FLUENT_ELASTICSEARCH_TARGET_INDEX_KEY'] || use_nil}" + type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}" + include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'false'}" + template_name "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_NAME'] || use_nil}" + template_file "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_FILE'] || use_nil}" + template_overwrite "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_OVERWRITE'] || use_default}" + sniffer_class_name "#{ENV['FLUENT_SNIFFER_CLASS_NAME'] || 'Fluent::Plugin::ElasticsearchSimpleSniffer'}" + request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '5s'}" + application_name "#{ENV['FLUENT_ELASTICSEARCH_APPLICATION_NAME'] || use_default}" + <buffer> + flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}" + flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}" + chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}" + queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}" + retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_forever true + </buffer> +</match> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/systemd.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/systemd.conf new file mode 100644 index 000000000..8256db2d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/systemd.conf @@ -0,0 +1,55 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/entrypoint.sh b/docker-image/v1.18/arm64/debian-elasticsearch8/entrypoint.sh new file mode 100755 index 000000000..f7820b6b5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +set -e + +SIMPLE_SNIFFER=$( gem contents fluent-plugin-elasticsearch | grep elasticsearch_simple_sniffer.rb ) + +if [ -n "$SIMPLE_SNIFFER" -a -f "$SIMPLE_SNIFFER" ] ; then + FLUENTD_OPT="$FLUENTD_OPT -r $SIMPLE_SNIFFER" +fi + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-elasticsearch8/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-elasticsearch8/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-elasticsearch8/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-elasticsearch8/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-elasticsearch8/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-forward/.dockerignore b/docker-image/v1.18/arm64/debian-forward/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-forward/Dockerfile b/docker-image/v1.18/arm64/debian-forward/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-forward/Gemfile b/docker-image/v1.18/arm64/debian-forward/Gemfile new file mode 100644 index 000000000..e079bb664 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/Gemfile @@ -0,0 +1,20 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-forward/Gemfile.lock b/docker-image/v1.18/arm64/debian-forward/Gemfile.lock new file mode 100644 index 000000000..c02b3672f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/Gemfile.lock @@ -0,0 +1,144 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-forward/conf/fluent.conf b/docker-image/v1.18/arm64/debian-forward/conf/fluent.conf new file mode 100644 index 000000000..da980fc79 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/fluent.conf @@ -0,0 +1,22 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type forward + @id out_fwd + @log_level info + <server> + host "#{ENV['FLUENT_FORWARD_HOST'] || ENV['FLUENT_FOWARD_HOST']}" + port "#{ENV['FLUENT_FORWARD_PORT'] || ENV['FLUENT_FOWARD_PORT']}" + </server> + <buffer> + flush_interval "#{ENV['FLUENT_FORWARD_FLUSH_INTERVAL'] || use_default}" + </buffer> +</match> + diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-forward/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-forward/conf/systemd.conf b/docker-image/v1.18/arm64/debian-forward/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-forward/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-forward/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-forward/entrypoint.sh b/docker-image/v1.18/arm64/debian-forward/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-forward/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-forward/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-forward/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-forward/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-forward/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-forward/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-forward/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-gcs/.dockerignore b/docker-image/v1.18/arm64/debian-gcs/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-gcs/Dockerfile b/docker-image/v1.18/arm64/debian-gcs/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-gcs/Gemfile b/docker-image/v1.18/arm64/debian-gcs/Gemfile new file mode 100644 index 000000000..e5f60ce7b --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-gcs", "0.4.2" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-gcs/Gemfile.lock b/docker-image/v1.18/arm64/debian-gcs/Gemfile.lock new file mode 100644 index 000000000..ef160e361 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/Gemfile.lock @@ -0,0 +1,215 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + declarative (0.0.20) + digest-crc (0.6.5) + rake (>= 12.0.0, < 14.0.0) + domain_name (0.6.20240107) + drb (2.2.1) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.0) + net-http (>= 0.5.0) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-gcs (0.4.2) + fluentd (>= 0.14.22, < 2) + google-cloud-storage (~> 1.1) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + google-apis-core (0.15.1) + addressable (~> 2.5, >= 2.5.1) + googleauth (~> 1.9) + httpclient (>= 2.8.3, < 3.a) + mini_mime (~> 1.0) + mutex_m + representable (~> 3.0) + retriable (>= 2.0, < 4.a) + google-apis-iamcredentials_v1 (0.22.0) + google-apis-core (>= 0.15.0, < 2.a) + google-apis-storage_v1 (0.49.0) + google-apis-core (>= 0.15.0, < 2.a) + google-cloud-core (1.7.1) + google-cloud-env (>= 1.0, < 3.a) + google-cloud-errors (~> 1.0) + google-cloud-env (2.2.1) + faraday (>= 1.0, < 3.a) + google-cloud-errors (1.4.0) + google-cloud-storage (1.54.0) + addressable (~> 2.8) + digest-crc (~> 0.4) + google-apis-core (~> 0.13) + google-apis-iamcredentials_v1 (~> 0.18) + google-apis-storage_v1 (~> 0.38) + google-cloud-core (~> 1.6) + googleauth (~> 1.9) + mini_mime (~> 1.0) + google-logging-utils (0.1.0) + googleauth (1.12.2) + faraday (>= 1.0, < 3.a) + google-cloud-env (~> 2.2) + google-logging-utils (~> 0.1) + jwt (>= 1.4, < 3.0) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (>= 0.16, < 2.a) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + httpclient (2.8.3) + json (2.9.1) + jsonpath (1.1.5) + multi_json + jwt (2.9.3) + base64 + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + mini_mime (1.1.5) + msgpack (1.7.5) + multi_json (1.15.0) + mutex_m (0.3.0) + net-http (0.6.0) + uri + netrc (0.11.0) + oj (3.15.1) + os (1.1.4) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + representable (3.2.0) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + retriable (3.1.2) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + signet (0.19.0) + addressable (~> 2.8) + faraday (>= 0.17.5, < 3.a) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + trailblazer-option (0.1.2) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + uber (0.1.0) + uri (1.0.2) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-gcs (= 0.4.2) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/fluent.conf b/docker-image/v1.18/arm64/debian-gcs/conf/fluent.conf new file mode 100644 index 000000000..2e28aebb4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/fluent.conf @@ -0,0 +1,23 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + # docs: https://github.com/daichirata/fluent-plugin-gcs + # this configuration relies on the nodes having permission to write on your gs bucket + @type gcs + @id out_gcs + project "#{ENV['GCS_BUCKET_PROJECT']}" + bucket "#{ENV['GCS_BUCKET_NAME']}" + object_key_format %{path}%{time_slice}/%{hostname}/%{index}.%{file_extension} + path logs/ + buffer_path /var/log/fluentd-buffers/gcs.buffer + time_slice_format %Y/%m/%d + time_slice_wait 10m + utc +</match> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-gcs/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/systemd.conf b/docker-image/v1.18/arm64/debian-gcs/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-gcs/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-gcs/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-gcs/entrypoint.sh b/docker-image/v1.18/arm64/debian-gcs/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-gcs/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-gcs/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-gcs/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-gcs/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-gcs/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-gcs/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-gcs/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-graylog/.dockerignore b/docker-image/v1.18/arm64/debian-graylog/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-graylog/Dockerfile b/docker-image/v1.18/arm64/debian-graylog/Dockerfile new file mode 100644 index 000000000..71401c6b2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential patch zlib1g-dev liblzma-dev git" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-graylog/Gemfile b/docker-image/v1.18/arm64/debian-graylog/Gemfile new file mode 100644 index 000000000..742425892 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "gelf" +gem "fluent-plugin-gelf-hs", "~> 1.0.7" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-graylog/Gemfile.lock b/docker-image/v1.18/arm64/debian-graylog/Gemfile.lock new file mode 100644 index 000000000..ab243a5c9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/Gemfile.lock @@ -0,0 +1,152 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-gelf-hs (1.0.8) + fluentd + gelf (>= 2.0.0) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + gelf (3.1.0) + json + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + json (2.9.1) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-gelf-hs (~> 1.0.7) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + gelf + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/fluent.conf b/docker-image/v1.18/arm64/debian-graylog/conf/fluent.conf new file mode 100644 index 000000000..5a89d11e4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/fluent.conf @@ -0,0 +1,33 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +# The gelf plugin assumes input in utf-8 +<filter **> + @type record_modifier + @id graylog_encode_utf8 + char_encoding utf-8 +</filter> + +<match **> + @type gelf + @id out_graylog + @log_level info + include_tag_key true + host "#{ENV['FLUENT_GRAYLOG_HOST']}" + port "#{ENV['FLUENT_GRAYLOG_PORT']}" + protocol "#{ENV['FLUENT_GRAYLOG_PROTOCOL'] || 'udp'}" + <buffer> + flush_thread_count 8 + flush_interval 5s + chunk_limit_size 8M + queue_limit_length 512 + retry_max_interval 30 + retry_forever true + </buffer> +</match> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-graylog/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/systemd.conf b/docker-image/v1.18/arm64/debian-graylog/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-graylog/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-graylog/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-graylog/entrypoint.sh b/docker-image/v1.18/arm64/debian-graylog/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-graylog/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-graylog/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-graylog/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-graylog/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-graylog/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-graylog/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-graylog/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-kafka/.dockerignore b/docker-image/v1.18/arm64/debian-kafka/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-kafka/Dockerfile b/docker-image/v1.18/arm64/debian-kafka/Dockerfile new file mode 100644 index 000000000..538e7ba07 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/Dockerfile @@ -0,0 +1,65 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev" \ + runtimeDeps="krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2" \ + && export DEBIAN_FRONTEND=noninteractive && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem \ + && ldd $(gem contents rdkafka | grep librdkafka.so) | grep libsasl2.so.2 + +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-kafka/Gemfile b/docker-image/v1.18/arm64/debian-kafka/Gemfile new file mode 100644 index 000000000..a2b26ff57 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/Gemfile @@ -0,0 +1,25 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "rdkafka", "~> 0.11.1" +gem "fluent-plugin-kafka", "~> 0.19.0" +gem "fluent-plugin-avro", "~> 1.1.1" +gem "fluent-plugin-parser-avro", "~> 0.3.1" +gem "snappy", "~> 0.0.15" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-kafka/Gemfile.lock b/docker-image/v1.18/arm64/debian-kafka/Gemfile.lock new file mode 100644 index 000000000..c64a4952d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/Gemfile.lock @@ -0,0 +1,172 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + avro (1.12.0) + multi_json (~> 1.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + digest-crc (0.6.5) + rake (>= 12.0.0, < 14.0.0) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-avro (1.1.1) + avro + fluentd (>= 0.14.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kafka (0.19.3) + fluentd (>= 0.10.58, < 2) + ltsv + ruby-kafka (>= 1.5.0, < 2) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-avro (0.3.1) + avro + fluentd (>= 0.14.10, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + ltsv (0.1.2) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + mini_portile2 (2.8.8) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + rdkafka (0.11.1) + ffi (~> 1.15) + mini_portile2 (~> 2.6) + rake (> 12) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + ruby-kafka (1.5.0) + digest-crc + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + snappy (0.0.17) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-avro (~> 1.1.1) + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kafka (~> 0.19.0) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-avro (~> 0.3.1) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rdkafka (~> 0.11.1) + rexml (~> 3.2.5) + snappy (~> 0.0.15) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/fluent.conf b/docker-image/v1.18/arm64/debian-kafka/conf/fluent.conf new file mode 100644 index 000000000..251c92d82 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/fluent.conf @@ -0,0 +1,33 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type kafka_buffered + @id out_kafka + + brokers "#{ENV['FLUENT_KAFKA_BROKERS']}" + + default_topic "#{ENV['FLUENT_KAFKA_DEFAULT_TOPIC'] || nil}" + default_partition_key "#{ENV['FLUENT_KAFKA_DEFAULT_PARTITION_KEY'] || nil}" + default_message_key "#{ENV['FLUENT_KAFKA_DEFAULT_MESSAGE_KEY'] || nil}" + output_data_type "#{ENV['FLUENT_KAFKA_OUTPUT_DATA_TYPE'] || 'json'}" + output_include_tag "#{ENV['FLUENT_KAFKA_OUTPUT_INCLUDE_TAG'] || false}" + output_include_time "#{ENV['FLUENT_KAFKA_OUTPUT_INCLUDE_TIME'] || false}" + exclude_topic_key "#{ENV['FLUENT_KAFKA_EXCLUDE_TOPIC_KEY'] || false}" + exclude_partition_key "#{ENV['FLUENT_KAFKA_EXCLUDE_PARTITION_KEY'] || false}" + get_kafka_client_log "#{ENV['FLUENT_KAFKA_GET_KAFKA_CLIENT_LOG'] || false}" + + # ruby-kafka producer options + max_send_retries "#{ENV['FLUENT_KAFKA_MAX_SEND_RETRIES'] || 1}" + required_acks "#{ENV['FLUENT_KAFKA_REQUIRED_ACKS'] || -1}" + ack_timeout "#{ENV['FLUENT_KAFKA_ACK_TIMEOUT'] || nil}" + compression_codec "#{ENV['FLUENT_KAFKA_COMPRESSION_CODEC'] || nil}" + max_send_limit_bytes "#{ENV['FLUENT_KAFKA_MAX_SEND_LIMIT_BYTES'] || nil}" + discard_kafka_delivery_failed "#{ENV['FLUENT_KAFKA_DISCARD_KAFKA_DELIVERY_FAILED'] || false}" +</match> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-kafka/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/systemd.conf b/docker-image/v1.18/arm64/debian-kafka/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-kafka/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-kafka/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-kafka/entrypoint.sh b/docker-image/v1.18/arm64/debian-kafka/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-kafka/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-kafka/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-kafka/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-kafka/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-kafka/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-kafka/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-kafka2/.dockerignore b/docker-image/v1.18/arm64/debian-kafka2/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-kafka2/Dockerfile b/docker-image/v1.18/arm64/debian-kafka2/Dockerfile new file mode 100644 index 000000000..538e7ba07 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/Dockerfile @@ -0,0 +1,65 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev" \ + runtimeDeps="krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2" \ + && export DEBIAN_FRONTEND=noninteractive && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem \ + && ldd $(gem contents rdkafka | grep librdkafka.so) | grep libsasl2.so.2 + +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-kafka2/Gemfile b/docker-image/v1.18/arm64/debian-kafka2/Gemfile new file mode 100644 index 000000000..a2b26ff57 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/Gemfile @@ -0,0 +1,25 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "rdkafka", "~> 0.11.1" +gem "fluent-plugin-kafka", "~> 0.19.0" +gem "fluent-plugin-avro", "~> 1.1.1" +gem "fluent-plugin-parser-avro", "~> 0.3.1" +gem "snappy", "~> 0.0.15" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-kafka2/Gemfile.lock b/docker-image/v1.18/arm64/debian-kafka2/Gemfile.lock new file mode 100644 index 000000000..c64a4952d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/Gemfile.lock @@ -0,0 +1,172 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + avro (1.12.0) + multi_json (~> 1.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + digest-crc (0.6.5) + rake (>= 12.0.0, < 14.0.0) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-avro (1.1.1) + avro + fluentd (>= 0.14.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kafka (0.19.3) + fluentd (>= 0.10.58, < 2) + ltsv + ruby-kafka (>= 1.5.0, < 2) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-avro (0.3.1) + avro + fluentd (>= 0.14.10, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + ltsv (0.1.2) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + mini_portile2 (2.8.8) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + rdkafka (0.11.1) + ffi (~> 1.15) + mini_portile2 (~> 2.6) + rake (> 12) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + ruby-kafka (1.5.0) + digest-crc + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + snappy (0.0.17) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-avro (~> 1.1.1) + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kafka (~> 0.19.0) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-avro (~> 0.3.1) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rdkafka (~> 0.11.1) + rexml (~> 3.2.5) + snappy (~> 0.0.15) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/fluent.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/fluent.conf new file mode 100644 index 000000000..8bfafe6a3 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/fluent.conf @@ -0,0 +1,37 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type kafka2 + @id out_kafka2 + + brokers "#{ENV['FLUENT_KAFKA2_BROKERS']}" + + default_topic "#{ENV['FLUENT_KAFKA2_DEFAULT_TOPIC'] || nil}" + default_partition_key "#{ENV['FLUENT_KAFKA2_DEFAULT_PARTITION_KEY'] || nil}" + default_message_key "#{ENV['FLUENT_KAFKA2_DEFAULT_MESSAGE_KEY'] || nil}" + exclude_topic_key "#{ENV['FLUENT_KAFKA2_EXCLUDE_TOPIC_KEY'] || false}" + exclude_partition_key "#{ENV['FLUENT_KAFKA2_EXCLUDE_PARTITION_KEY'] || false}" + get_kafka_client_log "#{ENV['FLUENT_KAFKA2_GET_KAFKA_CLIENT_LOG'] || false}" + <format> + @type "#{ENV['FLUENT_KAFKA2_OUTPUT_FORMAT_TYPE'] || 'json'}" + </format> + <inject> + tag_key "#{ENV['FLUENT_KAFKA2_OUTPUT_TAG_KEY'] || use_nil}" + time_key "#{ENV['FLUENT_KAFKA2_OUTPUT_TIME_KEY'] || use_nil}" + </inject> + + # ruby-kafka producer options + max_send_retries "#{ENV['FLUENT_KAFKA2_MAX_SEND_RETRIES'] || 1}" + required_acks "#{ENV['FLUENT_KAFKA2_REQUIRED_ACKS'] || -1}" + ack_timeout "#{ENV['FLUENT_KAFKA2_ACK_TIMEOUT'] || nil}" + compression_codec "#{ENV['FLUENT_KAFKA2_COMPRESSION_CODEC'] || nil}" + max_send_limit_bytes "#{ENV['FLUENT_KAFKA2_MAX_SEND_LIMIT_BYTES'] || nil}" + discard_kafka_delivery_failed "#{ENV['FLUENT_KAFKA2_DISCARD_KAFKA_DELIVERY_FAILED'] || false}" +</match> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/systemd.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-kafka2/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-kafka2/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-kafka2/entrypoint.sh b/docker-image/v1.18/arm64/debian-kafka2/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-kafka2/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-kafka2/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-kafka2/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-kafka2/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-kafka2/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-kafka2/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kafka2/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-kinesis/.dockerignore b/docker-image/v1.18/arm64/debian-kinesis/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-kinesis/Dockerfile b/docker-image/v1.18/arm64/debian-kinesis/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-kinesis/Gemfile b/docker-image/v1.18/arm64/debian-kinesis/Gemfile new file mode 100644 index 000000000..df47abeda --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-kinesis", "~> 3.4.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-kinesis/Gemfile.lock b/docker-image/v1.18/arm64/debian-kinesis/Gemfile.lock new file mode 100644 index 000000000..2a7aa38d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/Gemfile.lock @@ -0,0 +1,167 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.1026.0) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-firehose (1.85.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-kinesis (1.71.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kinesis (3.4.2) + aws-sdk-firehose (~> 1, != 1.9, != 1.5, != 1.15) + aws-sdk-kinesis (~> 1, != 1.5, != 1.4, != 1.14) + fluentd (>= 0.14.22, < 2) + google-protobuf (~> 3) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + google-protobuf (3.25.5) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jmespath (1.6.2) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kinesis (~> 3.4.0) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/fluent.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/fluent.conf new file mode 100644 index 000000000..a2b820c6a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/fluent.conf @@ -0,0 +1,24 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type kinesis_streams + @id out_kinesis_streams + region "#{ENV['FLUENT_KINESIS_STREAMS_REGION'] || nil}" + stream_name "#{ENV['FLUENT_KINESIS_STREAMS_STREAM_NAME']}" + include_time_key "#{ENV['FLUENT_KINESIS_STREAMS_INCLUDE_TIME_KEY'] || false}" + <buffer> + flush_interval 1 + chunk_limit_size "#{ENV['FLUENT_KINESIS_STREAMS_CHUNK_LIMIT_SIZE'] || '1m'}" + flush_thread_interval 0.1 + flush_thread_burst_interval 0.01 + flush_thread_count 15 + </buffer> +</match> + diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/systemd.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-kinesis/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-kinesis/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-kinesis/entrypoint.sh b/docker-image/v1.18/arm64/debian-kinesis/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-kinesis/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-kinesis/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-kinesis/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-kinesis/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-kinesis/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-kinesis/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-kinesis/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-logentries/.dockerignore b/docker-image/v1.18/arm64/debian-logentries/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-logentries/Dockerfile b/docker-image/v1.18/arm64/debian-logentries/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-logentries/Gemfile b/docker-image/v1.18/arm64/debian-logentries/Gemfile new file mode 100644 index 000000000..2d34048ba --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +#gem "fluent-plugin-logentries" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-logentries/Gemfile.lock b/docker-image/v1.18/arm64/debian-logentries/Gemfile.lock new file mode 100644 index 000000000..c02b3672f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/Gemfile.lock @@ -0,0 +1,144 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/fluent.conf b/docker-image/v1.18/arm64/debian-logentries/conf/fluent.conf new file mode 100644 index 000000000..145d90b41 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/fluent.conf @@ -0,0 +1,17 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type logentries + @id out_logentries + use_json true + tag_access_log stdout + tag_error_log stderr + config_path /etc/logentries/tokens.yaml +</match> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-logentries/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/systemd.conf b/docker-image/v1.18/arm64/debian-logentries/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-logentries/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-logentries/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-logentries/entrypoint.sh b/docker-image/v1.18/arm64/debian-logentries/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-logentries/plugins/out_logentries.rb b/docker-image/v1.18/arm64/debian-logentries/plugins/out_logentries.rb new file mode 100644 index 000000000..d83f52035 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/plugins/out_logentries.rb @@ -0,0 +1,153 @@ +# via https://www.fluentd.org/plugins/all#input-output + +# Usage: https://github.com/Woorank/fluent-plugin-logentries#configruation-file-yml +require 'socket' +require 'yaml' +require 'openssl' +require 'fluent/output' + +class Fluent::LogentriesOutput < Fluent::BufferedOutput + class ConnectionFailure < StandardError; end + # First, register the plugin. NAME is the name of this plugin + # and identifies the plugin in the configuration file. + Fluent::Plugin.register_output('logentries', self) + + config_param :use_ssl, :bool, :default => true + config_param :use_json, :bool, :default => false + config_param :port, :integer, :default => 20000 + config_param :protocol, :string, :default => 'tcp' + config_param :config_path, :string + config_param :max_retries, :integer, :default => 3 + config_param :tag_access_log, :string, :default => 'logs-access' + config_param :tag_error_log, :string, :default => 'logs-error' + config_param :default_token, :string, :default => nil + + SSL_HOST = "api.logentries.com" + NO_SSL_HOST = "data.logentries.com" + + def configure(conf) + super + + @tokens = nil + @last_edit = Time.at(0) + end + + def start + super + end + + def shutdown + super + end + + def client + @_socket ||= if @use_ssl + context = OpenSSL::SSL::SSLContext.new + socket = TCPSocket.new SSL_HOST, @port + ssl_client = OpenSSL::SSL::SSLSocket.new socket, context + + ssl_client.connect + else + if @protocol == 'tcp' + TCPSocket.new NO_SSL_HOST, @port + else + udp_client = UDPSocket.new + udp_client.connect NO_SSL_HOST, @port + + udp_client + end + end + end + + # This method is called when an event reaches Fluentd. + def format(tag, time, record) + return [tag, record].to_msgpack + end + + # Parse an YML file and generate a list of tokens. + # It will only re-generate the list on changes. + def generate_tokens_list + begin + edit_time = File.mtime(@config_path) + + if edit_time > @last_edit + @tokens = YAML::load_file(@config_path) + @last_edit = edit_time + + log.info "Token(s) list updated." + end + rescue Exception => e + log.warn "Could not load configuration. #{e.message}" + end + end + + # Returns the correct token to use for a given tag / records + def get_token(tag, record) + app_name = record["app_name"] || '' + + # Config Structure + # ----------------------- + # app-name: + # app: TOKEN + # access: TOKEN (optional) + # error: TOKEN (optional) + @tokens.each do |key, value| + if app_name == key || tag.index(key) != nil + default = value['app'] + + case tag + when @tag_access_log + return value['access'] || default + when @tag_error_log + return value['error'] || default + + else + return default + end + end + end + + return default_token + end + + # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins. + def write(chunk) + generate_tokens_list() + return unless @tokens.is_a? Hash + + chunk.msgpack_each do |tag, record| + next unless record.is_a? Hash + next unless @use_json or record.has_key? "message" + + token = get_token(tag, record) + next if token.nil? + + # Clean up the string to avoid blank line in logentries + message = @use_json ? record.to_json : record["message"].rstrip() + send_logentries(token, message) + end + end + + def send_logentries(token, data) + retries = 0 + begin + client.write("#{token} #{data} \n") + rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT, Errno::EPIPE => e + if retries < @max_retries + retries += 1 + @_socket = nil + log.warn "Could not push logs to Logentries, resetting connection and trying again. #{e.message}" + sleep 5**retries + retry + end + raise ConnectionFailure, "Could not push logs to Logentries after #{retries} retries. #{e.message}" + rescue Errno::EMSGSIZE + str_length = data.length + send_logentries(token, data[0..str_length/2]) + send_logentries(token, data[(str_length/2)+1..str_length]) + + log.warn "Message Too Long, re-sending it in two part..." + end + end + +end diff --git a/docker-image/v1.18/arm64/debian-logentries/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-logentries/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-logentries/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-logentries/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logentries/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-loggly/.dockerignore b/docker-image/v1.18/arm64/debian-loggly/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-loggly/Dockerfile b/docker-image/v1.18/arm64/debian-loggly/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-loggly/Gemfile b/docker-image/v1.18/arm64/debian-loggly/Gemfile new file mode 100644 index 000000000..5cf9e0cab --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-loggly", "~> 0.0.9" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-loggly/Gemfile.lock b/docker-image/v1.18/arm64/debian-loggly/Gemfile.lock new file mode 100644 index 000000000..3158a8aee --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/Gemfile.lock @@ -0,0 +1,149 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-loggly (0.0.9) + net-http-persistent (~> 2.7) + yajl-ruby (~> 1.0) + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + net-http-persistent (2.9.4) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-loggly (~> 0.0.9) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/fluent.conf b/docker-image/v1.18/arm64/debian-loggly/conf/fluent.conf new file mode 100644 index 000000000..1155cceb7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/fluent.conf @@ -0,0 +1,15 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type loggly + @id out_loggly + @log_level info + loggly_url "https://logs-01.loggly.com/bulk/#{ENV['LOGGLY_TOKEN']}/tag/#{ENV['LOGGLY_TAGS'] || 'fluentd'}/bulk" +</match> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-loggly/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/systemd.conf b/docker-image/v1.18/arm64/debian-loggly/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-loggly/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-loggly/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-loggly/entrypoint.sh b/docker-image/v1.18/arm64/debian-loggly/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-loggly/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-loggly/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-loggly/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-loggly/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-loggly/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-loggly/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-loggly/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-logzio/.dockerignore b/docker-image/v1.18/arm64/debian-logzio/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-logzio/Dockerfile b/docker-image/v1.18/arm64/debian-logzio/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-logzio/Gemfile b/docker-image/v1.18/arm64/debian-logzio/Gemfile new file mode 100644 index 000000000..1cac0733d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-logzio", "~> 0.2.2" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-logzio/Gemfile.lock b/docker-image/v1.18/arm64/debian-logzio/Gemfile.lock new file mode 100644 index 000000000..309e61db7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/Gemfile.lock @@ -0,0 +1,152 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + connection_pool (2.4.1) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-logzio (0.2.2) + fluentd (>= 0.14.0, < 2) + net-http-persistent (~> 4.0) + prometheus-client (>= 2.1.0) + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + net-http-persistent (4.0.5) + connection_pool (~> 2.2) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-logzio (~> 0.2.2) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/fluent.conf b/docker-image/v1.18/arm64/debian-logzio/conf/fluent.conf new file mode 100644 index 000000000..a8633d84e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/fluent.conf @@ -0,0 +1,39 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type logzio_buffered + @id out_logzio + endpoint_url "https://listener.logz.io:8071?token=#{ENV['LOGZIO_TOKEN']}&type=#{ENV['LOGZIO_LOGTYPE']}" + output_include_time true + output_include_tags true + <buffer> + # Set the buffer type to file to improve the reliability and reduce the memory consumption + @type file + path /var/log/fluentd-buffers/stackdriver.buffer + # Set queue_full action to block because we want to pause gracefully + # in case of the off-the-limits load instead of throwing an exception + overflow_action block + # Set the chunk limit conservatively to avoid exceeding the GCL limit + # of 10MiB per write request. + chunk_limit_size 2M + # Cap the combined memory usage of this buffer and the one below to + # 2MiB/chunk * (6 + 2) chunks = 16 MiB + queue_limit_length 6 + # Never wait more than 5 seconds before flushing logs in the non-error case. + flush_interval 5s + # Never wait longer than 30 seconds between retries. + retry_max_interval 30 + # Disable the limit on the number of retries (retry forever). + retry_forever true + # Use multiple threads for processing. + flush_thread_count 2 + </buffer> +</match> + diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-logzio/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/systemd.conf b/docker-image/v1.18/arm64/debian-logzio/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-logzio/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-logzio/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-logzio/entrypoint.sh b/docker-image/v1.18/arm64/debian-logzio/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-logzio/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-logzio/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-logzio/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-logzio/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-logzio/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-logzio/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-logzio/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-opensearch/.dockerignore b/docker-image/v1.18/arm64/debian-opensearch/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-opensearch/Dockerfile b/docker-image/v1.18/arm64/debian-opensearch/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-opensearch/Gemfile b/docker-image/v1.18/arm64/debian-opensearch/Gemfile new file mode 100644 index 000000000..924560249 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-opensearch", "~> 1.1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-opensearch/Gemfile.lock b/docker-image/v1.18/arm64/debian-opensearch/Gemfile.lock new file mode 100644 index 000000000..31c9dab19 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/Gemfile.lock @@ -0,0 +1,183 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.1026.0) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + excon (1.2.2) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-excon (2.3.0) + excon (>= 1.0.0) + faraday (>= 2.11.0, < 3) + faraday-net_http (3.4.0) + net-http (>= 0.5.0) + faraday_middleware-aws-sigv4 (1.0.1) + aws-sigv4 (~> 1.0) + faraday (>= 2.0, < 3) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-opensearch (1.1.5) + aws-sdk-core (~> 3) + excon + faraday (>= 2.0.0) + faraday-excon (>= 2.0.0) + faraday_middleware-aws-sigv4 (~> 1.0.1) + fluentd (>= 0.14.22) + opensearch-ruby (>= 3.0.1) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jmespath (1.6.2) + json (2.9.1) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + net-http (0.6.0) + uri + netrc (0.11.0) + oj (3.15.1) + opensearch-ruby (3.4.0) + faraday (>= 1.0, < 3) + multi_json (>= 1.0) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + uri (1.0.2) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-opensearch (~> 1.1.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/fluent.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/fluent.conf new file mode 100644 index 000000000..8cc30ff75 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/fluent.conf @@ -0,0 +1,42 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type opensearch + @id out_os + @log_level info + include_tag_key true + host "#{ENV['FLUENT_OPENSEARCH_HOST']}" + port "#{ENV['FLUENT_OPENSEARCH_PORT']}" + path "#{ENV['FLUENT_OPENSEARCH_PATH']}" + scheme "#{ENV['FLUENT_OPENSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_OPENSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_OPENSEARCH_SSL_VERSION'] || 'TLSv1_2'}" + ca_file "#{ENV['FLUENT_OPENSEARCH_CA_PATH']}" + user "#{ENV['FLUENT_OPENSEARCH_USER']}" + password "#{ENV['FLUENT_OPENSEARCH_PASSWORD']}" + client_cert "#{ENV['FLUENT_OPENSEARCH_CLIENT_CERT']}" + client_key "#{ENV['FLUENT_OPENSEARCH_CLIENT_KEY']}" + client_key_pass "#{ENV['FLUENT_OPENSEARCH_CLIENT_KEY_PASS']}" + index_name "#{ENV['FLUENT_OPENSEARCH_INDEX_NAME'] || 'fluentd'}" + logstash_dateformat "#{ENV['FLUENT_OPENSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" + logstash_format "#{ENV['FLUENT_OPENSEARCH_LOGSTASH_FORMAT'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_OPENSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" + logstash_prefix_separator "#{ENV['FLUENT_OPENSEARCH_LOGSTASH_PREFIX_SEPARATOR'] || '-'}" + <buffer> + flush_thread_count "#{ENV['FLUENT_OPENSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '1'}" + flush_mode "#{ENV['FLUENT_OPENSEARCH_BUFFER_FLUSH_MODE'] || 'interval'}" + flush_interval "#{ENV['FLUENT_OPENSEARCH_BUFFER_FLUSH_INTERVAL'] || '60s'}" + chunk_limit_size "#{ENV['FLUENT_OPENSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '8M'}" + total_limit_size "#{ENV['FLUENT_OPENSEARCH_BUFFER_TOTAL_LIMIT_SIZE'] || '512M'}" + retry_max_interval "#{ENV['FLUENT_OPENSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_timeout "#{ENV['FLUENT_OPENSEARCH_BUFFER_RETRY_TIMEOUT'] || '72h'}" + retry_forever "#{ENV['FLUENT_OPENSEARCH_BUFFER_RETRY_FOREVER'] || 'false'}" + </buffer> +</match> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/systemd.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-opensearch/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-opensearch/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-opensearch/entrypoint.sh b/docker-image/v1.18/arm64/debian-opensearch/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-opensearch/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-opensearch/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-opensearch/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-opensearch/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-opensearch/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-opensearch/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-opensearch/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-papertrail/.dockerignore b/docker-image/v1.18/arm64/debian-papertrail/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-papertrail/Dockerfile b/docker-image/v1.18/arm64/debian-papertrail/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-papertrail/Gemfile b/docker-image/v1.18/arm64/debian-papertrail/Gemfile new file mode 100644 index 000000000..c3cd2b0a3 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-papertrail", "~> 0.2.6" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-papertrail/Gemfile.lock b/docker-image/v1.18/arm64/debian-papertrail/Gemfile.lock new file mode 100644 index 000000000..963554a1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/Gemfile.lock @@ -0,0 +1,152 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-mixin-config-placeholders (0.4.0) + fluentd + uuidtools (>= 2.1.5) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-papertrail (0.2.8) + fluent-mixin-config-placeholders (~> 0.4.0) + fluentd (>= 0.10, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + uuidtools (2.2.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-papertrail (~> 0.2.6) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/fluent.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/fluent.conf new file mode 100644 index 000000000..8b0cc403c --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/fluent.conf @@ -0,0 +1,26 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + + +## Capture audit logs +#<match kube-apiserver-audit> +# @type papertrail +# +# papertrail_host "#{ENV['FLUENT_PAPERTRAIL_AUDIT_HOST']}" +# papertrail_port "#{ENV['FLUENT_PAPERTRAIL_AUDIT_PORT']}" +#</match> + +<match **> + @type papertrail + @id out_papertrail + + papertrail_host "#{ENV['FLUENT_PAPERTRAIL_HOST']}" + papertrail_port "#{ENV['FLUENT_PAPERTRAIL_PORT']}" + +</match> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes.conf new file mode 100644 index 000000000..68fc59c5e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes.conf @@ -0,0 +1,64 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + +<filter kube-apiserver-audit> + @type record_transformer + @id filter_rt_kube_apiserver_audit + enable_ruby true + <record> + hostname #{ENV['FLUENT_HOSTNAME']} + program kube-apiserver-audit + severity info + facility local0 + message ${record} + </record> +</filter> + +<filter kubernetes.**> + @type record_transformer + @id filter_rt_kube_logs + enable_ruby true + <record> + hostname #{ENV['FLUENT_HOSTNAME']}-${record["kubernetes"]["namespace_name"]}-${record["kubernetes"]["pod_name"]} + program ${record["kubernetes"]["container_name"]} + severity info + facility local0 + message ${record['log'] || record['message']} + </record> +</filter> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/systemd.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/systemd.conf new file mode 100644 index 000000000..01ce25607 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/systemd.conf @@ -0,0 +1,58 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + +<filter bootkube> + @type record_transformer + @id filter_rt_bootkube + enable_ruby true + <record> + hostname #{ENV['FLUENT_HOSTNAME']} + program kube-bootkube + severity info + facility local0 + message ${record['log']} + </record> +</filter> diff --git a/docker-image/v1.18/arm64/debian-papertrail/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-papertrail/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-papertrail/entrypoint.sh b/docker-image/v1.18/arm64/debian-papertrail/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-papertrail/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-papertrail/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-papertrail/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-papertrail/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-papertrail/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-papertrail/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-papertrail/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-s3/.dockerignore b/docker-image/v1.18/arm64/debian-s3/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-s3/Dockerfile b/docker-image/v1.18/arm64/debian-s3/Dockerfile new file mode 100644 index 000000000..7012dbe57 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/Dockerfile @@ -0,0 +1,67 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev curl" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && curl -sL -o columnify_0.1.0_Linux_x86_64.tar.gz https://github.com/reproio/columnify/releases/download/v0.1.0/columnify_0.1.0_Linux_x86_64.tar.gz \ + && tar xf columnify_0.1.0_Linux_x86_64.tar.gz \ + && rm LICENSE README.md columnify_0.1.0_Linux_x86_64.tar.gz \ + && mv columnify /usr/local/bin/ \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-s3/Gemfile b/docker-image/v1.18/arm64/debian-s3/Gemfile new file mode 100644 index 000000000..e1a584542 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "aws-sdk-s3", "~> 1.101" +gem "fluent-plugin-s3", "~> 1.7.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-s3/Gemfile.lock b/docker-image/v1.18/arm64/debian-s3/Gemfile.lock new file mode 100644 index 000000000..5cb99d70e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/Gemfile.lock @@ -0,0 +1,170 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.1026.0) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.96.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-s3 (1.176.1) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + aws-sdk-sqs (1.89.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-s3 (1.7.2) + aws-sdk-s3 (~> 1.60) + aws-sdk-sqs (~> 1.23) + fluentd (>= 0.14.22, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jmespath (1.6.2) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + aws-sdk-s3 (~> 1.101) + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-s3 (~> 1.7.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-s3/conf/fluent.conf b/docker-image/v1.18/arm64/debian-s3/conf/fluent.conf new file mode 100644 index 000000000..98dc47d3a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/fluent.conf @@ -0,0 +1,31 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + # docs: https://docs.fluentd.org/v0.12/articles/out_s3 + # note: this configuration relies on the nodes have an IAM instance profile with access to your S3 bucket + @type s3 + @id out_s3 + @log_level info + s3_bucket "#{ENV['S3_BUCKET_NAME']}" + s3_region "#{ENV['S3_BUCKET_REGION']}" + s3_object_key_format "#{ENV['S3_OBJECT_KEY_FORMAT'] || '%{path}%Y/%m/%d/cluster-log-%{index}.%{file_extension}'}" + <inject> + time_key time + tag_key tag + localtime false + </inject> + <buffer> + @type file + path /var/log/fluentd-buffers/s3.buffer + timekey "#{ENV['S3_TIMEKEY'] || '3600'}" + timekey_use_utc true + chunk_limit_size "#{ENV['S3_CHUNK_LIMIT_SIZE'] || '256m'}" + </buffer> +</match> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-s3/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-s3/conf/systemd.conf b/docker-image/v1.18/arm64/debian-s3/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-s3/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-s3/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-s3/entrypoint.sh b/docker-image/v1.18/arm64/debian-s3/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-s3/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-s3/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-s3/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-s3/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-s3/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-s3/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-s3/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-syslog/.dockerignore b/docker-image/v1.18/arm64/debian-syslog/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/arm64/debian-syslog/Dockerfile b/docker-image/v1.18/arm64/debian-syslog/Dockerfile new file mode 100644 index 000000000..ccefe99a5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/Dockerfile @@ -0,0 +1,63 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +# For multiarch build on Docker hub automated build. +FROM golang:alpine AS builder +WORKDIR /go +ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8 +RUN apk add curl --no-cache +RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static . + +FROM fluent/fluentd:v1.18.0-debian-arm64-1.0 +COPY --from=builder /go/qemu-aarch64-static /usr/bin/ + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/arm64/debian-syslog/Gemfile b/docker-image/v1.18/arm64/debian-syslog/Gemfile new file mode 100644 index 000000000..d7e4f38a4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-remote_syslog" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/arm64/debian-syslog/Gemfile.lock b/docker-image/v1.18/arm64/debian-syslog/Gemfile.lock new file mode 100644 index 000000000..09a305f5d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/Gemfile.lock @@ -0,0 +1,151 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-remote_syslog (1.1.0) + fluentd + remote_syslog_sender (>= 1.1.1) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + remote_syslog_sender (1.2.2) + syslog_protocol + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + syslog_protocol (0.9.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-remote_syslog + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/fluent.conf b/docker-image/v1.18/arm64/debian-syslog/conf/fluent.conf new file mode 100644 index 000000000..941d44f57 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/fluent.conf @@ -0,0 +1,33 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type remote_syslog + @id out_kube_remote_syslog + host "#{ENV['SYSLOG_HOST']}" + port "#{ENV['SYSLOG_PORT']}" + severity debug + program fluentd + hostname ${kubernetes_host} + + protocol "#{ENV['SYSLOG_PROTOCOL'] || 'tcp'}" + tls "#{ENV['SYSLOG_TLS'] || 'false'}" + ca_file "#{ENV['SYSLOG_CA_FILE'] || ''}" + verify_mode "#{ENV['SYSLOG_VERIFY_MODE'] || ''}" + packet_size 65535 + + <buffer kubernetes_host> + flush_interval "#{ENV['FLUENT_SYSLOG_FLUSH_INTERVAL'] || use_default}" + </buffer> + + <format> + @type ltsv + </format> +</match> + diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/containers.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/docker.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/etcd.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/glbc.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kubelet.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/salt.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/startupscript.conf b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/prometheus.conf b/docker-image/v1.18/arm64/debian-syslog/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/systemd.conf b/docker-image/v1.18/arm64/debian-syslog/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/arm64/debian-syslog/conf/tail_container_parse.conf b/docker-image/v1.18/arm64/debian-syslog/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/arm64/debian-syslog/entrypoint.sh b/docker-image/v1.18/arm64/debian-syslog/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/arm64/debian-syslog/plugins/.gitkeep b/docker-image/v1.18/arm64/debian-syslog/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/arm64/debian-syslog/plugins/parser_kubernetes.rb b/docker-image/v1.18/arm64/debian-syslog/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/arm64/debian-syslog/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/arm64/debian-syslog/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/arm64/debian-syslog/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-azureblob/.dockerignore b/docker-image/v1.18/debian-azureblob/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-azureblob/Dockerfile b/docker-image/v1.18/debian-azureblob/Dockerfile new file mode 100644 index 000000000..ab3c529f2 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="pkg-config libxslt-dev libxml2-dev" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true && bundle config build.nokogiri --use-system-libraries \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-azureblob/Gemfile b/docker-image/v1.18/debian-azureblob/Gemfile new file mode 100644 index 000000000..bc2bf565d --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "azure-storage-blob", "~> 2.0" +gem "fluent-plugin-azure-storage-append-blob-lts", "~> 0.7.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-azureblob/Gemfile.lock b/docker-image/v1.18/debian-azureblob/Gemfile.lock new file mode 100644 index 000000000..bba5d39ff --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/Gemfile.lock @@ -0,0 +1,192 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + azure-storage-blob (2.0.3) + azure-storage-common (~> 2.0) + nokogiri (~> 1, >= 1.10.8) + azure-storage-common (2.0.4) + faraday (~> 1.0) + faraday_middleware (~> 1.0, >= 1.0.0.rc1) + net-http-persistent (~> 4.0) + nokogiri (~> 1, >= 1.10.8) + base64 (0.2.0) + concurrent-ruby (1.3.4) + connection_pool (2.4.1) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + faraday (1.10.4) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.1.0) + multipart-post (~> 2.0) + faraday-net_http (1.0.2) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + faraday_middleware (1.2.1) + faraday (~> 1.0) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-azure-storage-append-blob-lts (0.7.0) + azure-storage-blob (~> 2.0) + fluentd (>= 0.14.10, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + mini_portile2 (2.8.8) + msgpack (1.7.5) + multi_json (1.15.0) + multipart-post (2.4.1) + net-http-persistent (4.0.5) + connection_pool (~> 2.2) + netrc (0.11.0) + nokogiri (1.17.2) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + racc (1.8.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + ruby2_keywords (0.0.5) + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + azure-storage-blob (~> 2.0) + ffi + fluent-plugin-azure-storage-append-blob-lts (~> 0.7.0) + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-azureblob/conf/fluent.conf b/docker-image/v1.18/debian-azureblob/conf/fluent.conf new file mode 100644 index 000000000..a0e4ae0b1 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/fluent.conf @@ -0,0 +1,36 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type azure-storage-append-blob + @id out_azure_storage_append_blob + + azure_cloud "#{ENV['AZUREBLOB_AZURE_CLOUD']}" + azure_storage_account "#{ENV['AZUREBLOB_ACCOUNT_NAME']}" + azure_storage_access_key "#{ENV['AZUREBLOB_ACCOUNT_KEY']}" + azure_storage_connection_string "#{ENV['AZUREBLOB_CONNECTION_STRING']}" + azure_storage_sas_token "#{ENV['AZUREBLOB_SAS_TOKEN']}" + azure_container "#{ENV['AZUREBLOB_CONTAINER']}" + azure_imds_api_version "#{ENV['AZUREBLOB_IMDS_API_VERSION']}" + azure_token_refresh_interval "#{ENV['AZUREBLOB_TOKEN_REFRESH_INTERVAL']}" + auto_create_container true + path "#{ENV['AZUREBLOB_LOG_PATH']}" + azure_object_key_format %{path}%{time_slice}_%{index}.log + time_slice_format %Y%m%d-%H + # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format, + # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument. + <buffer> + @type file + path /var/log/fluent/azurestorageappendblob + timekey 60 # 1 minute + timekey_wait 60 + timekey_use_utc true # use utc + chunk_limit_size 256m + </buffer> +</match> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-azureblob/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/prometheus.conf b/docker-image/v1.18/debian-azureblob/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-azureblob/conf/systemd.conf b/docker-image/v1.18/debian-azureblob/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-azureblob/conf/tail_container_parse.conf b/docker-image/v1.18/debian-azureblob/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-azureblob/entrypoint.sh b/docker-image/v1.18/debian-azureblob/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-azureblob/plugins/.gitkeep b/docker-image/v1.18/debian-azureblob/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-azureblob/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-azureblob/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-azureblob/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-azureblob/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-azureblob/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-cloudwatch/.dockerignore b/docker-image/v1.18/debian-cloudwatch/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-cloudwatch/Dockerfile b/docker-image/v1.18/debian-cloudwatch/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-cloudwatch/Gemfile b/docker-image/v1.18/debian-cloudwatch/Gemfile new file mode 100644 index 000000000..c184080da --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "aws-sdk-cloudwatchlogs", "~> 1.0" +gem "fluent-plugin-cloudwatch-logs", "~> 0.14.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-cloudwatch/Gemfile.lock b/docker-image/v1.18/debian-cloudwatch/Gemfile.lock new file mode 100644 index 000000000..870022c20 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/Gemfile.lock @@ -0,0 +1,162 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.1026.0) + aws-sdk-cloudwatchlogs (1.105.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-cloudwatch-logs (0.14.3) + aws-sdk-cloudwatchlogs (~> 1.0) + fluentd (>= 1.8.0) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jmespath (1.6.2) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + aws-sdk-cloudwatchlogs (~> 1.0) + ffi + fluent-plugin-cloudwatch-logs (~> 0.14.0) + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-cloudwatch/conf/fluent.conf b/docker-image/v1.18/debian-cloudwatch/conf/fluent.conf new file mode 100644 index 000000000..d676cd1f6 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/fluent.conf @@ -0,0 +1,19 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type cloudwatch_logs + @id out_cloudwatch_logs + log_group_name "#{ENV['LOG_GROUP_NAME']}" + auto_create_stream true + use_tag_as_stream true + retention_in_days "#{ENV['RETENTION_IN_DAYS'] || 'nil'}" + json_handler yajl # To avoid UndefinedConversionError + log_rejected_request "#{ENV['LOG_REJECTED_REQUEST']}" # Log rejected request for missing parts +</match> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/prometheus.conf b/docker-image/v1.18/debian-cloudwatch/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-cloudwatch/conf/systemd.conf b/docker-image/v1.18/debian-cloudwatch/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-cloudwatch/conf/tail_container_parse.conf b/docker-image/v1.18/debian-cloudwatch/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-cloudwatch/entrypoint.sh b/docker-image/v1.18/debian-cloudwatch/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-cloudwatch/plugins/.gitkeep b/docker-image/v1.18/debian-cloudwatch/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-cloudwatch/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-cloudwatch/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-cloudwatch/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-cloudwatch/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-cloudwatch/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-elasticsearch7/.dockerignore b/docker-image/v1.18/debian-elasticsearch7/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-elasticsearch7/Dockerfile b/docker-image/v1.18/debian-elasticsearch7/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-elasticsearch7/Gemfile b/docker-image/v1.18/debian-elasticsearch7/Gemfile new file mode 100644 index 000000000..4ba394d64 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/Gemfile @@ -0,0 +1,24 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "elasticsearch", "~> 7.0" +gem "fluent-plugin-elasticsearch", "~> 5.2.5" +gem "elasticsearch-xpack", "~> 7.0" +gem "fluent-plugin-dedot_filter", "~> 1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-elasticsearch7/Gemfile.lock b/docker-image/v1.18/debian-elasticsearch7/Gemfile.lock new file mode 100644 index 000000000..eafa70014 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/Gemfile.lock @@ -0,0 +1,192 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + elasticsearch (7.17.11) + elasticsearch-api (= 7.17.11) + elasticsearch-transport (= 7.17.11) + elasticsearch-api (7.17.11) + multi_json + elasticsearch-transport (7.17.11) + base64 + faraday (>= 1, < 3) + multi_json + elasticsearch-xpack (7.17.11) + elasticsearch-api (>= 6) + excon (1.2.2) + faraday (1.10.4) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.1.0) + multipart-post (~> 2.0) + faraday-net_http (1.0.2) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-dedot_filter (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-elasticsearch (5.2.5) + elasticsearch + excon + faraday (~> 1.10) + fluentd (>= 0.14.22) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + multipart-post (2.4.1) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + ruby2_keywords (0.0.5) + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + elasticsearch (~> 7.0) + elasticsearch-xpack (~> 7.0) + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-dedot_filter (~> 1.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-elasticsearch (~> 5.2.5) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/fluent.conf b/docker-image/v1.18/debian-elasticsearch7/conf/fluent.conf new file mode 100644 index 000000000..feb7607c9 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/fluent.conf @@ -0,0 +1,53 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type elasticsearch + @id out_es + @log_level info + include_tag_key true + host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}" + path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}" + scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1_2'}" + user "#{ENV['FLUENT_ELASTICSEARCH_USER'] || use_default}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD'] || use_default}" + reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}" + reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}" + reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}" + log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" + logstash_dateformat "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" + logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}" + index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}" + target_index_key "#{ENV['FLUENT_ELASTICSEARCH_TARGET_INDEX_KEY'] || use_nil}" + type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}" + include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'false'}" + template_name "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_NAME'] || use_nil}" + template_file "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_FILE'] || use_nil}" + template_overwrite "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_OVERWRITE'] || use_default}" + sniffer_class_name "#{ENV['FLUENT_SNIFFER_CLASS_NAME'] || 'Fluent::Plugin::ElasticsearchSimpleSniffer'}" + request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '5s'}" + application_name "#{ENV['FLUENT_ELASTICSEARCH_APPLICATION_NAME'] || use_default}" + suppress_type_name "#{ENV['FLUENT_ELASTICSEARCH_SUPPRESS_TYPE_NAME'] || 'true'}" + enable_ilm "#{ENV['FLUENT_ELASTICSEARCH_ENABLE_ILM'] || 'false'}" + ilm_policy_id "#{ENV['FLUENT_ELASTICSEARCH_ILM_POLICY_ID'] || use_default}" + ilm_policy "#{ENV['FLUENT_ELASTICSEARCH_ILM_POLICY'] || use_default}" + ilm_policy_overwrite "#{ENV['FLUENT_ELASTICSEARCH_ILM_POLICY_OVERWRITE'] || 'false'}" + <buffer> + flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}" + flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}" + chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}" + queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}" + retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_forever true + </buffer> +</match> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/prometheus.conf b/docker-image/v1.18/debian-elasticsearch7/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/systemd.conf b/docker-image/v1.18/debian-elasticsearch7/conf/systemd.conf new file mode 100644 index 000000000..8256db2d6 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/systemd.conf @@ -0,0 +1,55 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-elasticsearch7/conf/tail_container_parse.conf b/docker-image/v1.18/debian-elasticsearch7/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-elasticsearch7/entrypoint.sh b/docker-image/v1.18/debian-elasticsearch7/entrypoint.sh new file mode 100755 index 000000000..f7820b6b5 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +set -e + +SIMPLE_SNIFFER=$( gem contents fluent-plugin-elasticsearch | grep elasticsearch_simple_sniffer.rb ) + +if [ -n "$SIMPLE_SNIFFER" -a -f "$SIMPLE_SNIFFER" ] ; then + FLUENTD_OPT="$FLUENTD_OPT -r $SIMPLE_SNIFFER" +fi + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-elasticsearch7/plugins/.gitkeep b/docker-image/v1.18/debian-elasticsearch7/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-elasticsearch7/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-elasticsearch7/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-elasticsearch7/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-elasticsearch7/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch7/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-elasticsearch8/.dockerignore b/docker-image/v1.18/debian-elasticsearch8/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-elasticsearch8/Dockerfile b/docker-image/v1.18/debian-elasticsearch8/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-elasticsearch8/Gemfile b/docker-image/v1.18/debian-elasticsearch8/Gemfile new file mode 100644 index 000000000..66a9c8df8 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/Gemfile @@ -0,0 +1,23 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "elasticsearch", "~> 8.0" +gem "fluent-plugin-elasticsearch", "~> 5.3.0" +gem "fluent-plugin-dedot_filter", "~> 1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-elasticsearch8/Gemfile.lock b/docker-image/v1.18/debian-elasticsearch8/Gemfile.lock new file mode 100644 index 000000000..608f651f9 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/Gemfile.lock @@ -0,0 +1,177 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + elastic-transport (8.3.5) + faraday (< 3) + multi_json + elasticsearch (8.17.0) + elastic-transport (~> 8.3) + elasticsearch-api (= 8.17.0) + elasticsearch-api (8.17.0) + multi_json + excon (1.2.2) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-excon (2.3.0) + excon (>= 1.0.0) + faraday (>= 2.11.0, < 3) + faraday-net_http (3.4.0) + net-http (>= 0.5.0) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-dedot_filter (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-elasticsearch (5.3.0) + elasticsearch + excon + faraday (>= 2.0.0) + faraday-excon (>= 2.0.0) + fluentd (>= 0.14.22) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + json (2.9.1) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + net-http (0.6.0) + uri + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + uri (1.0.2) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + elasticsearch (~> 8.0) + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-dedot_filter (~> 1.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-elasticsearch (~> 5.3.0) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/fluent.conf b/docker-image/v1.18/debian-elasticsearch8/conf/fluent.conf new file mode 100644 index 000000000..6a8a8b322 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/fluent.conf @@ -0,0 +1,48 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type elasticsearch + @id out_es + @log_level info + include_tag_key true + host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}" + port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}" + path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}" + scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}" + user "#{ENV['FLUENT_ELASTICSEARCH_USER'] || use_default}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD'] || use_default}" + reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}" + reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}" + reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}" + log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" + logstash_dateformat "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" + logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}" + index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}" + target_index_key "#{ENV['FLUENT_ELASTICSEARCH_TARGET_INDEX_KEY'] || use_nil}" + type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}" + include_timestamp "#{ENV['FLUENT_ELASTICSEARCH_INCLUDE_TIMESTAMP'] || 'false'}" + template_name "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_NAME'] || use_nil}" + template_file "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_FILE'] || use_nil}" + template_overwrite "#{ENV['FLUENT_ELASTICSEARCH_TEMPLATE_OVERWRITE'] || use_default}" + sniffer_class_name "#{ENV['FLUENT_SNIFFER_CLASS_NAME'] || 'Fluent::Plugin::ElasticsearchSimpleSniffer'}" + request_timeout "#{ENV['FLUENT_ELASTICSEARCH_REQUEST_TIMEOUT'] || '5s'}" + application_name "#{ENV['FLUENT_ELASTICSEARCH_APPLICATION_NAME'] || use_default}" + <buffer> + flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}" + flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}" + chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}" + queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}" + retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_forever true + </buffer> +</match> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/prometheus.conf b/docker-image/v1.18/debian-elasticsearch8/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/systemd.conf b/docker-image/v1.18/debian-elasticsearch8/conf/systemd.conf new file mode 100644 index 000000000..8256db2d6 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/systemd.conf @@ -0,0 +1,55 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + <entry> + fields_strip_underscores true + </entry> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-elasticsearch8/conf/tail_container_parse.conf b/docker-image/v1.18/debian-elasticsearch8/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-elasticsearch8/entrypoint.sh b/docker-image/v1.18/debian-elasticsearch8/entrypoint.sh new file mode 100755 index 000000000..f7820b6b5 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +set -e + +SIMPLE_SNIFFER=$( gem contents fluent-plugin-elasticsearch | grep elasticsearch_simple_sniffer.rb ) + +if [ -n "$SIMPLE_SNIFFER" -a -f "$SIMPLE_SNIFFER" ] ; then + FLUENTD_OPT="$FLUENTD_OPT -r $SIMPLE_SNIFFER" +fi + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-elasticsearch8/plugins/.gitkeep b/docker-image/v1.18/debian-elasticsearch8/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-elasticsearch8/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-elasticsearch8/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-elasticsearch8/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-elasticsearch8/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-elasticsearch8/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-forward/.dockerignore b/docker-image/v1.18/debian-forward/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-forward/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-forward/Dockerfile b/docker-image/v1.18/debian-forward/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-forward/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-forward/Gemfile b/docker-image/v1.18/debian-forward/Gemfile new file mode 100644 index 000000000..e079bb664 --- /dev/null +++ b/docker-image/v1.18/debian-forward/Gemfile @@ -0,0 +1,20 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-forward/Gemfile.lock b/docker-image/v1.18/debian-forward/Gemfile.lock new file mode 100644 index 000000000..c02b3672f --- /dev/null +++ b/docker-image/v1.18/debian-forward/Gemfile.lock @@ -0,0 +1,144 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-forward/conf/fluent.conf b/docker-image/v1.18/debian-forward/conf/fluent.conf new file mode 100644 index 000000000..da980fc79 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/fluent.conf @@ -0,0 +1,22 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type forward + @id out_fwd + @log_level info + <server> + host "#{ENV['FLUENT_FORWARD_HOST'] || ENV['FLUENT_FOWARD_HOST']}" + port "#{ENV['FLUENT_FORWARD_PORT'] || ENV['FLUENT_FOWARD_PORT']}" + </server> + <buffer> + flush_interval "#{ENV['FLUENT_FORWARD_FLUSH_INTERVAL'] || use_default}" + </buffer> +</match> + diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes.conf b/docker-image/v1.18/debian-forward/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-forward/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-forward/conf/prometheus.conf b/docker-image/v1.18/debian-forward/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-forward/conf/systemd.conf b/docker-image/v1.18/debian-forward/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-forward/conf/tail_container_parse.conf b/docker-image/v1.18/debian-forward/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-forward/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-forward/entrypoint.sh b/docker-image/v1.18/debian-forward/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-forward/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-forward/plugins/.gitkeep b/docker-image/v1.18/debian-forward/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-forward/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-forward/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-forward/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-forward/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-forward/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-forward/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-gcs/.dockerignore b/docker-image/v1.18/debian-gcs/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-gcs/Dockerfile b/docker-image/v1.18/debian-gcs/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-gcs/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-gcs/Gemfile b/docker-image/v1.18/debian-gcs/Gemfile new file mode 100644 index 000000000..e5f60ce7b --- /dev/null +++ b/docker-image/v1.18/debian-gcs/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-gcs", "0.4.2" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-gcs/Gemfile.lock b/docker-image/v1.18/debian-gcs/Gemfile.lock new file mode 100644 index 000000000..ef160e361 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/Gemfile.lock @@ -0,0 +1,215 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + declarative (0.0.20) + digest-crc (0.6.5) + rake (>= 12.0.0, < 14.0.0) + domain_name (0.6.20240107) + drb (2.2.1) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.0) + net-http (>= 0.5.0) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-gcs (0.4.2) + fluentd (>= 0.14.22, < 2) + google-cloud-storage (~> 1.1) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + google-apis-core (0.15.1) + addressable (~> 2.5, >= 2.5.1) + googleauth (~> 1.9) + httpclient (>= 2.8.3, < 3.a) + mini_mime (~> 1.0) + mutex_m + representable (~> 3.0) + retriable (>= 2.0, < 4.a) + google-apis-iamcredentials_v1 (0.22.0) + google-apis-core (>= 0.15.0, < 2.a) + google-apis-storage_v1 (0.49.0) + google-apis-core (>= 0.15.0, < 2.a) + google-cloud-core (1.7.1) + google-cloud-env (>= 1.0, < 3.a) + google-cloud-errors (~> 1.0) + google-cloud-env (2.2.1) + faraday (>= 1.0, < 3.a) + google-cloud-errors (1.4.0) + google-cloud-storage (1.54.0) + addressable (~> 2.8) + digest-crc (~> 0.4) + google-apis-core (~> 0.13) + google-apis-iamcredentials_v1 (~> 0.18) + google-apis-storage_v1 (~> 0.38) + google-cloud-core (~> 1.6) + googleauth (~> 1.9) + mini_mime (~> 1.0) + google-logging-utils (0.1.0) + googleauth (1.12.2) + faraday (>= 1.0, < 3.a) + google-cloud-env (~> 2.2) + google-logging-utils (~> 0.1) + jwt (>= 1.4, < 3.0) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (>= 0.16, < 2.a) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + httpclient (2.8.3) + json (2.9.1) + jsonpath (1.1.5) + multi_json + jwt (2.9.3) + base64 + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + mini_mime (1.1.5) + msgpack (1.7.5) + multi_json (1.15.0) + mutex_m (0.3.0) + net-http (0.6.0) + uri + netrc (0.11.0) + oj (3.15.1) + os (1.1.4) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + representable (3.2.0) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + retriable (3.1.2) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + signet (0.19.0) + addressable (~> 2.8) + faraday (>= 0.17.5, < 3.a) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + trailblazer-option (0.1.2) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + uber (0.1.0) + uri (1.0.2) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-gcs (= 0.4.2) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-gcs/conf/fluent.conf b/docker-image/v1.18/debian-gcs/conf/fluent.conf new file mode 100644 index 000000000..2e28aebb4 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/fluent.conf @@ -0,0 +1,23 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + # docs: https://github.com/daichirata/fluent-plugin-gcs + # this configuration relies on the nodes having permission to write on your gs bucket + @type gcs + @id out_gcs + project "#{ENV['GCS_BUCKET_PROJECT']}" + bucket "#{ENV['GCS_BUCKET_NAME']}" + object_key_format %{path}%{time_slice}/%{hostname}/%{index}.%{file_extension} + path logs/ + buffer_path /var/log/fluentd-buffers/gcs.buffer + time_slice_format %Y/%m/%d + time_slice_wait 10m + utc +</match> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-gcs/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/prometheus.conf b/docker-image/v1.18/debian-gcs/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-gcs/conf/systemd.conf b/docker-image/v1.18/debian-gcs/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-gcs/conf/tail_container_parse.conf b/docker-image/v1.18/debian-gcs/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-gcs/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-gcs/entrypoint.sh b/docker-image/v1.18/debian-gcs/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-gcs/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-gcs/plugins/.gitkeep b/docker-image/v1.18/debian-gcs/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-gcs/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-gcs/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-gcs/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-gcs/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-gcs/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-gcs/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-graylog/.dockerignore b/docker-image/v1.18/debian-graylog/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-graylog/Dockerfile b/docker-image/v1.18/debian-graylog/Dockerfile new file mode 100644 index 000000000..fab9a7901 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential patch zlib1g-dev liblzma-dev git" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-graylog/Gemfile b/docker-image/v1.18/debian-graylog/Gemfile new file mode 100644 index 000000000..742425892 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "gelf" +gem "fluent-plugin-gelf-hs", "~> 1.0.7" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-graylog/Gemfile.lock b/docker-image/v1.18/debian-graylog/Gemfile.lock new file mode 100644 index 000000000..ab243a5c9 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/Gemfile.lock @@ -0,0 +1,152 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-gelf-hs (1.0.8) + fluentd + gelf (>= 2.0.0) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + gelf (3.1.0) + json + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + json (2.9.1) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-gelf-hs (~> 1.0.7) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + gelf + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-graylog/conf/fluent.conf b/docker-image/v1.18/debian-graylog/conf/fluent.conf new file mode 100644 index 000000000..5a89d11e4 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/fluent.conf @@ -0,0 +1,33 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +# The gelf plugin assumes input in utf-8 +<filter **> + @type record_modifier + @id graylog_encode_utf8 + char_encoding utf-8 +</filter> + +<match **> + @type gelf + @id out_graylog + @log_level info + include_tag_key true + host "#{ENV['FLUENT_GRAYLOG_HOST']}" + port "#{ENV['FLUENT_GRAYLOG_PORT']}" + protocol "#{ENV['FLUENT_GRAYLOG_PROTOCOL'] || 'udp'}" + <buffer> + flush_thread_count 8 + flush_interval 5s + chunk_limit_size 8M + queue_limit_length 512 + retry_max_interval 30 + retry_forever true + </buffer> +</match> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-graylog/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/prometheus.conf b/docker-image/v1.18/debian-graylog/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-graylog/conf/systemd.conf b/docker-image/v1.18/debian-graylog/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-graylog/conf/tail_container_parse.conf b/docker-image/v1.18/debian-graylog/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-graylog/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-graylog/entrypoint.sh b/docker-image/v1.18/debian-graylog/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-graylog/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-graylog/plugins/.gitkeep b/docker-image/v1.18/debian-graylog/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-graylog/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-graylog/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-graylog/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-graylog/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-graylog/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-graylog/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-kafka/.dockerignore b/docker-image/v1.18/debian-kafka/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-kafka/Dockerfile b/docker-image/v1.18/debian-kafka/Dockerfile new file mode 100644 index 000000000..85255027f --- /dev/null +++ b/docker-image/v1.18/debian-kafka/Dockerfile @@ -0,0 +1,57 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev" \ + runtimeDeps="krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2" \ + && export DEBIAN_FRONTEND=noninteractive && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem \ + && ldd $(gem contents rdkafka | grep librdkafka.so) | grep libsasl2.so.2 + +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-kafka/Gemfile b/docker-image/v1.18/debian-kafka/Gemfile new file mode 100644 index 000000000..a2b26ff57 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/Gemfile @@ -0,0 +1,25 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "rdkafka", "~> 0.11.1" +gem "fluent-plugin-kafka", "~> 0.19.0" +gem "fluent-plugin-avro", "~> 1.1.1" +gem "fluent-plugin-parser-avro", "~> 0.3.1" +gem "snappy", "~> 0.0.15" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-kafka/Gemfile.lock b/docker-image/v1.18/debian-kafka/Gemfile.lock new file mode 100644 index 000000000..c64a4952d --- /dev/null +++ b/docker-image/v1.18/debian-kafka/Gemfile.lock @@ -0,0 +1,172 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + avro (1.12.0) + multi_json (~> 1.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + digest-crc (0.6.5) + rake (>= 12.0.0, < 14.0.0) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-avro (1.1.1) + avro + fluentd (>= 0.14.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kafka (0.19.3) + fluentd (>= 0.10.58, < 2) + ltsv + ruby-kafka (>= 1.5.0, < 2) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-avro (0.3.1) + avro + fluentd (>= 0.14.10, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + ltsv (0.1.2) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + mini_portile2 (2.8.8) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + rdkafka (0.11.1) + ffi (~> 1.15) + mini_portile2 (~> 2.6) + rake (> 12) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + ruby-kafka (1.5.0) + digest-crc + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + snappy (0.0.17) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-avro (~> 1.1.1) + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kafka (~> 0.19.0) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-avro (~> 0.3.1) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rdkafka (~> 0.11.1) + rexml (~> 3.2.5) + snappy (~> 0.0.15) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-kafka/conf/fluent.conf b/docker-image/v1.18/debian-kafka/conf/fluent.conf new file mode 100644 index 000000000..251c92d82 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/fluent.conf @@ -0,0 +1,33 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type kafka_buffered + @id out_kafka + + brokers "#{ENV['FLUENT_KAFKA_BROKERS']}" + + default_topic "#{ENV['FLUENT_KAFKA_DEFAULT_TOPIC'] || nil}" + default_partition_key "#{ENV['FLUENT_KAFKA_DEFAULT_PARTITION_KEY'] || nil}" + default_message_key "#{ENV['FLUENT_KAFKA_DEFAULT_MESSAGE_KEY'] || nil}" + output_data_type "#{ENV['FLUENT_KAFKA_OUTPUT_DATA_TYPE'] || 'json'}" + output_include_tag "#{ENV['FLUENT_KAFKA_OUTPUT_INCLUDE_TAG'] || false}" + output_include_time "#{ENV['FLUENT_KAFKA_OUTPUT_INCLUDE_TIME'] || false}" + exclude_topic_key "#{ENV['FLUENT_KAFKA_EXCLUDE_TOPIC_KEY'] || false}" + exclude_partition_key "#{ENV['FLUENT_KAFKA_EXCLUDE_PARTITION_KEY'] || false}" + get_kafka_client_log "#{ENV['FLUENT_KAFKA_GET_KAFKA_CLIENT_LOG'] || false}" + + # ruby-kafka producer options + max_send_retries "#{ENV['FLUENT_KAFKA_MAX_SEND_RETRIES'] || 1}" + required_acks "#{ENV['FLUENT_KAFKA_REQUIRED_ACKS'] || -1}" + ack_timeout "#{ENV['FLUENT_KAFKA_ACK_TIMEOUT'] || nil}" + compression_codec "#{ENV['FLUENT_KAFKA_COMPRESSION_CODEC'] || nil}" + max_send_limit_bytes "#{ENV['FLUENT_KAFKA_MAX_SEND_LIMIT_BYTES'] || nil}" + discard_kafka_delivery_failed "#{ENV['FLUENT_KAFKA_DISCARD_KAFKA_DELIVERY_FAILED'] || false}" +</match> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-kafka/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/prometheus.conf b/docker-image/v1.18/debian-kafka/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-kafka/conf/systemd.conf b/docker-image/v1.18/debian-kafka/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-kafka/conf/tail_container_parse.conf b/docker-image/v1.18/debian-kafka/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-kafka/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-kafka/entrypoint.sh b/docker-image/v1.18/debian-kafka/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-kafka/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-kafka/plugins/.gitkeep b/docker-image/v1.18/debian-kafka/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-kafka/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-kafka/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-kafka/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-kafka/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-kafka/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-kafka/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-kafka2/.dockerignore b/docker-image/v1.18/debian-kafka2/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-kafka2/Dockerfile b/docker-image/v1.18/debian-kafka2/Dockerfile new file mode 100644 index 000000000..85255027f --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/Dockerfile @@ -0,0 +1,57 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev" \ + runtimeDeps="krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2" \ + && export DEBIAN_FRONTEND=noninteractive && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem \ + && ldd $(gem contents rdkafka | grep librdkafka.so) | grep libsasl2.so.2 + +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-kafka2/Gemfile b/docker-image/v1.18/debian-kafka2/Gemfile new file mode 100644 index 000000000..a2b26ff57 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/Gemfile @@ -0,0 +1,25 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "rdkafka", "~> 0.11.1" +gem "fluent-plugin-kafka", "~> 0.19.0" +gem "fluent-plugin-avro", "~> 1.1.1" +gem "fluent-plugin-parser-avro", "~> 0.3.1" +gem "snappy", "~> 0.0.15" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-kafka2/Gemfile.lock b/docker-image/v1.18/debian-kafka2/Gemfile.lock new file mode 100644 index 000000000..c64a4952d --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/Gemfile.lock @@ -0,0 +1,172 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + avro (1.12.0) + multi_json (~> 1.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + digest-crc (0.6.5) + rake (>= 12.0.0, < 14.0.0) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-avro (1.1.1) + avro + fluentd (>= 0.14.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kafka (0.19.3) + fluentd (>= 0.10.58, < 2) + ltsv + ruby-kafka (>= 1.5.0, < 2) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-avro (0.3.1) + avro + fluentd (>= 0.14.10, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + ltsv (0.1.2) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + mini_portile2 (2.8.8) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + rdkafka (0.11.1) + ffi (~> 1.15) + mini_portile2 (~> 2.6) + rake (> 12) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + ruby-kafka (1.5.0) + digest-crc + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + snappy (0.0.17) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-avro (~> 1.1.1) + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kafka (~> 0.19.0) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-avro (~> 0.3.1) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rdkafka (~> 0.11.1) + rexml (~> 3.2.5) + snappy (~> 0.0.15) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-kafka2/conf/fluent.conf b/docker-image/v1.18/debian-kafka2/conf/fluent.conf new file mode 100644 index 000000000..8bfafe6a3 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/fluent.conf @@ -0,0 +1,37 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type kafka2 + @id out_kafka2 + + brokers "#{ENV['FLUENT_KAFKA2_BROKERS']}" + + default_topic "#{ENV['FLUENT_KAFKA2_DEFAULT_TOPIC'] || nil}" + default_partition_key "#{ENV['FLUENT_KAFKA2_DEFAULT_PARTITION_KEY'] || nil}" + default_message_key "#{ENV['FLUENT_KAFKA2_DEFAULT_MESSAGE_KEY'] || nil}" + exclude_topic_key "#{ENV['FLUENT_KAFKA2_EXCLUDE_TOPIC_KEY'] || false}" + exclude_partition_key "#{ENV['FLUENT_KAFKA2_EXCLUDE_PARTITION_KEY'] || false}" + get_kafka_client_log "#{ENV['FLUENT_KAFKA2_GET_KAFKA_CLIENT_LOG'] || false}" + <format> + @type "#{ENV['FLUENT_KAFKA2_OUTPUT_FORMAT_TYPE'] || 'json'}" + </format> + <inject> + tag_key "#{ENV['FLUENT_KAFKA2_OUTPUT_TAG_KEY'] || use_nil}" + time_key "#{ENV['FLUENT_KAFKA2_OUTPUT_TIME_KEY'] || use_nil}" + </inject> + + # ruby-kafka producer options + max_send_retries "#{ENV['FLUENT_KAFKA2_MAX_SEND_RETRIES'] || 1}" + required_acks "#{ENV['FLUENT_KAFKA2_REQUIRED_ACKS'] || -1}" + ack_timeout "#{ENV['FLUENT_KAFKA2_ACK_TIMEOUT'] || nil}" + compression_codec "#{ENV['FLUENT_KAFKA2_COMPRESSION_CODEC'] || nil}" + max_send_limit_bytes "#{ENV['FLUENT_KAFKA2_MAX_SEND_LIMIT_BYTES'] || nil}" + discard_kafka_delivery_failed "#{ENV['FLUENT_KAFKA2_DISCARD_KAFKA_DELIVERY_FAILED'] || false}" +</match> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-kafka2/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/prometheus.conf b/docker-image/v1.18/debian-kafka2/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-kafka2/conf/systemd.conf b/docker-image/v1.18/debian-kafka2/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-kafka2/conf/tail_container_parse.conf b/docker-image/v1.18/debian-kafka2/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-kafka2/entrypoint.sh b/docker-image/v1.18/debian-kafka2/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-kafka2/plugins/.gitkeep b/docker-image/v1.18/debian-kafka2/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-kafka2/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-kafka2/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-kafka2/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-kafka2/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-kafka2/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-kinesis/.dockerignore b/docker-image/v1.18/debian-kinesis/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-kinesis/Dockerfile b/docker-image/v1.18/debian-kinesis/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-kinesis/Gemfile b/docker-image/v1.18/debian-kinesis/Gemfile new file mode 100644 index 000000000..df47abeda --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-kinesis", "~> 3.4.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-kinesis/Gemfile.lock b/docker-image/v1.18/debian-kinesis/Gemfile.lock new file mode 100644 index 000000000..2a7aa38d6 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/Gemfile.lock @@ -0,0 +1,167 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.1026.0) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-firehose (1.85.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-kinesis (1.71.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kinesis (3.4.2) + aws-sdk-firehose (~> 1, != 1.9, != 1.5, != 1.15) + aws-sdk-kinesis (~> 1, != 1.5, != 1.4, != 1.14) + fluentd (>= 0.14.22, < 2) + google-protobuf (~> 3) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + google-protobuf (3.25.5) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jmespath (1.6.2) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kinesis (~> 3.4.0) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-kinesis/conf/fluent.conf b/docker-image/v1.18/debian-kinesis/conf/fluent.conf new file mode 100644 index 000000000..a2b820c6a --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/fluent.conf @@ -0,0 +1,24 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type kinesis_streams + @id out_kinesis_streams + region "#{ENV['FLUENT_KINESIS_STREAMS_REGION'] || nil}" + stream_name "#{ENV['FLUENT_KINESIS_STREAMS_STREAM_NAME']}" + include_time_key "#{ENV['FLUENT_KINESIS_STREAMS_INCLUDE_TIME_KEY'] || false}" + <buffer> + flush_interval 1 + chunk_limit_size "#{ENV['FLUENT_KINESIS_STREAMS_CHUNK_LIMIT_SIZE'] || '1m'}" + flush_thread_interval 0.1 + flush_thread_burst_interval 0.01 + flush_thread_count 15 + </buffer> +</match> + diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-kinesis/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/prometheus.conf b/docker-image/v1.18/debian-kinesis/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-kinesis/conf/systemd.conf b/docker-image/v1.18/debian-kinesis/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-kinesis/conf/tail_container_parse.conf b/docker-image/v1.18/debian-kinesis/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-kinesis/entrypoint.sh b/docker-image/v1.18/debian-kinesis/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-kinesis/plugins/.gitkeep b/docker-image/v1.18/debian-kinesis/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-kinesis/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-kinesis/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-kinesis/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-kinesis/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-kinesis/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-logentries/.dockerignore b/docker-image/v1.18/debian-logentries/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-logentries/Dockerfile b/docker-image/v1.18/debian-logentries/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-logentries/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-logentries/Gemfile b/docker-image/v1.18/debian-logentries/Gemfile new file mode 100644 index 000000000..2d34048ba --- /dev/null +++ b/docker-image/v1.18/debian-logentries/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +#gem "fluent-plugin-logentries" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-logentries/Gemfile.lock b/docker-image/v1.18/debian-logentries/Gemfile.lock new file mode 100644 index 000000000..c02b3672f --- /dev/null +++ b/docker-image/v1.18/debian-logentries/Gemfile.lock @@ -0,0 +1,144 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-logentries/conf/fluent.conf b/docker-image/v1.18/debian-logentries/conf/fluent.conf new file mode 100644 index 000000000..145d90b41 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/fluent.conf @@ -0,0 +1,17 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type logentries + @id out_logentries + use_json true + tag_access_log stdout + tag_error_log stderr + config_path /etc/logentries/tokens.yaml +</match> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-logentries/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/prometheus.conf b/docker-image/v1.18/debian-logentries/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-logentries/conf/systemd.conf b/docker-image/v1.18/debian-logentries/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-logentries/conf/tail_container_parse.conf b/docker-image/v1.18/debian-logentries/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-logentries/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-logentries/entrypoint.sh b/docker-image/v1.18/debian-logentries/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-logentries/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-logentries/plugins/out_logentries.rb b/docker-image/v1.18/debian-logentries/plugins/out_logentries.rb new file mode 100644 index 000000000..d83f52035 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/plugins/out_logentries.rb @@ -0,0 +1,153 @@ +# via https://www.fluentd.org/plugins/all#input-output + +# Usage: https://github.com/Woorank/fluent-plugin-logentries#configruation-file-yml +require 'socket' +require 'yaml' +require 'openssl' +require 'fluent/output' + +class Fluent::LogentriesOutput < Fluent::BufferedOutput + class ConnectionFailure < StandardError; end + # First, register the plugin. NAME is the name of this plugin + # and identifies the plugin in the configuration file. + Fluent::Plugin.register_output('logentries', self) + + config_param :use_ssl, :bool, :default => true + config_param :use_json, :bool, :default => false + config_param :port, :integer, :default => 20000 + config_param :protocol, :string, :default => 'tcp' + config_param :config_path, :string + config_param :max_retries, :integer, :default => 3 + config_param :tag_access_log, :string, :default => 'logs-access' + config_param :tag_error_log, :string, :default => 'logs-error' + config_param :default_token, :string, :default => nil + + SSL_HOST = "api.logentries.com" + NO_SSL_HOST = "data.logentries.com" + + def configure(conf) + super + + @tokens = nil + @last_edit = Time.at(0) + end + + def start + super + end + + def shutdown + super + end + + def client + @_socket ||= if @use_ssl + context = OpenSSL::SSL::SSLContext.new + socket = TCPSocket.new SSL_HOST, @port + ssl_client = OpenSSL::SSL::SSLSocket.new socket, context + + ssl_client.connect + else + if @protocol == 'tcp' + TCPSocket.new NO_SSL_HOST, @port + else + udp_client = UDPSocket.new + udp_client.connect NO_SSL_HOST, @port + + udp_client + end + end + end + + # This method is called when an event reaches Fluentd. + def format(tag, time, record) + return [tag, record].to_msgpack + end + + # Parse an YML file and generate a list of tokens. + # It will only re-generate the list on changes. + def generate_tokens_list + begin + edit_time = File.mtime(@config_path) + + if edit_time > @last_edit + @tokens = YAML::load_file(@config_path) + @last_edit = edit_time + + log.info "Token(s) list updated." + end + rescue Exception => e + log.warn "Could not load configuration. #{e.message}" + end + end + + # Returns the correct token to use for a given tag / records + def get_token(tag, record) + app_name = record["app_name"] || '' + + # Config Structure + # ----------------------- + # app-name: + # app: TOKEN + # access: TOKEN (optional) + # error: TOKEN (optional) + @tokens.each do |key, value| + if app_name == key || tag.index(key) != nil + default = value['app'] + + case tag + when @tag_access_log + return value['access'] || default + when @tag_error_log + return value['error'] || default + + else + return default + end + end + end + + return default_token + end + + # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins. + def write(chunk) + generate_tokens_list() + return unless @tokens.is_a? Hash + + chunk.msgpack_each do |tag, record| + next unless record.is_a? Hash + next unless @use_json or record.has_key? "message" + + token = get_token(tag, record) + next if token.nil? + + # Clean up the string to avoid blank line in logentries + message = @use_json ? record.to_json : record["message"].rstrip() + send_logentries(token, message) + end + end + + def send_logentries(token, data) + retries = 0 + begin + client.write("#{token} #{data} \n") + rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT, Errno::EPIPE => e + if retries < @max_retries + retries += 1 + @_socket = nil + log.warn "Could not push logs to Logentries, resetting connection and trying again. #{e.message}" + sleep 5**retries + retry + end + raise ConnectionFailure, "Could not push logs to Logentries after #{retries} retries. #{e.message}" + rescue Errno::EMSGSIZE + str_length = data.length + send_logentries(token, data[0..str_length/2]) + send_logentries(token, data[(str_length/2)+1..str_length]) + + log.warn "Message Too Long, re-sending it in two part..." + end + end + +end diff --git a/docker-image/v1.18/debian-logentries/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-logentries/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-logentries/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-logentries/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-logentries/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-logentries/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-loggly/.dockerignore b/docker-image/v1.18/debian-loggly/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-loggly/Dockerfile b/docker-image/v1.18/debian-loggly/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-loggly/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-loggly/Gemfile b/docker-image/v1.18/debian-loggly/Gemfile new file mode 100644 index 000000000..5cf9e0cab --- /dev/null +++ b/docker-image/v1.18/debian-loggly/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-loggly", "~> 0.0.9" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-loggly/Gemfile.lock b/docker-image/v1.18/debian-loggly/Gemfile.lock new file mode 100644 index 000000000..3158a8aee --- /dev/null +++ b/docker-image/v1.18/debian-loggly/Gemfile.lock @@ -0,0 +1,149 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-loggly (0.0.9) + net-http-persistent (~> 2.7) + yajl-ruby (~> 1.0) + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + net-http-persistent (2.9.4) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-loggly (~> 0.0.9) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-loggly/conf/fluent.conf b/docker-image/v1.18/debian-loggly/conf/fluent.conf new file mode 100644 index 000000000..1155cceb7 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/fluent.conf @@ -0,0 +1,15 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type loggly + @id out_loggly + @log_level info + loggly_url "https://logs-01.loggly.com/bulk/#{ENV['LOGGLY_TOKEN']}/tag/#{ENV['LOGGLY_TAGS'] || 'fluentd'}/bulk" +</match> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-loggly/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/prometheus.conf b/docker-image/v1.18/debian-loggly/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-loggly/conf/systemd.conf b/docker-image/v1.18/debian-loggly/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-loggly/conf/tail_container_parse.conf b/docker-image/v1.18/debian-loggly/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-loggly/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-loggly/entrypoint.sh b/docker-image/v1.18/debian-loggly/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-loggly/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-loggly/plugins/.gitkeep b/docker-image/v1.18/debian-loggly/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-loggly/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-loggly/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-loggly/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-loggly/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-loggly/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-loggly/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-logzio/.dockerignore b/docker-image/v1.18/debian-logzio/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-logzio/Dockerfile b/docker-image/v1.18/debian-logzio/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-logzio/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-logzio/Gemfile b/docker-image/v1.18/debian-logzio/Gemfile new file mode 100644 index 000000000..1cac0733d --- /dev/null +++ b/docker-image/v1.18/debian-logzio/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-logzio", "~> 0.2.2" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-logzio/Gemfile.lock b/docker-image/v1.18/debian-logzio/Gemfile.lock new file mode 100644 index 000000000..309e61db7 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/Gemfile.lock @@ -0,0 +1,152 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + connection_pool (2.4.1) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-logzio (0.2.2) + fluentd (>= 0.14.0, < 2) + net-http-persistent (~> 4.0) + prometheus-client (>= 2.1.0) + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + net-http-persistent (4.0.5) + connection_pool (~> 2.2) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-logzio (~> 0.2.2) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-logzio/conf/fluent.conf b/docker-image/v1.18/debian-logzio/conf/fluent.conf new file mode 100644 index 000000000..a8633d84e --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/fluent.conf @@ -0,0 +1,39 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type logzio_buffered + @id out_logzio + endpoint_url "https://listener.logz.io:8071?token=#{ENV['LOGZIO_TOKEN']}&type=#{ENV['LOGZIO_LOGTYPE']}" + output_include_time true + output_include_tags true + <buffer> + # Set the buffer type to file to improve the reliability and reduce the memory consumption + @type file + path /var/log/fluentd-buffers/stackdriver.buffer + # Set queue_full action to block because we want to pause gracefully + # in case of the off-the-limits load instead of throwing an exception + overflow_action block + # Set the chunk limit conservatively to avoid exceeding the GCL limit + # of 10MiB per write request. + chunk_limit_size 2M + # Cap the combined memory usage of this buffer and the one below to + # 2MiB/chunk * (6 + 2) chunks = 16 MiB + queue_limit_length 6 + # Never wait more than 5 seconds before flushing logs in the non-error case. + flush_interval 5s + # Never wait longer than 30 seconds between retries. + retry_max_interval 30 + # Disable the limit on the number of retries (retry forever). + retry_forever true + # Use multiple threads for processing. + flush_thread_count 2 + </buffer> +</match> + diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-logzio/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/prometheus.conf b/docker-image/v1.18/debian-logzio/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-logzio/conf/systemd.conf b/docker-image/v1.18/debian-logzio/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-logzio/conf/tail_container_parse.conf b/docker-image/v1.18/debian-logzio/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-logzio/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-logzio/entrypoint.sh b/docker-image/v1.18/debian-logzio/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-logzio/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-logzio/plugins/.gitkeep b/docker-image/v1.18/debian-logzio/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-logzio/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-logzio/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-logzio/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-logzio/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-logzio/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-logzio/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-opensearch/.dockerignore b/docker-image/v1.18/debian-opensearch/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-opensearch/Dockerfile b/docker-image/v1.18/debian-opensearch/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-opensearch/Gemfile b/docker-image/v1.18/debian-opensearch/Gemfile new file mode 100644 index 000000000..924560249 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-opensearch", "~> 1.1.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-opensearch/Gemfile.lock b/docker-image/v1.18/debian-opensearch/Gemfile.lock new file mode 100644 index 000000000..31c9dab19 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/Gemfile.lock @@ -0,0 +1,183 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.1026.0) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + excon (1.2.2) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-excon (2.3.0) + excon (>= 1.0.0) + faraday (>= 2.11.0, < 3) + faraday-net_http (3.4.0) + net-http (>= 0.5.0) + faraday_middleware-aws-sigv4 (1.0.1) + aws-sigv4 (~> 1.0) + faraday (>= 2.0, < 3) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-opensearch (1.1.5) + aws-sdk-core (~> 3) + excon + faraday (>= 2.0.0) + faraday-excon (>= 2.0.0) + faraday_middleware-aws-sigv4 (~> 1.0.1) + fluentd (>= 0.14.22) + opensearch-ruby (>= 3.0.1) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jmespath (1.6.2) + json (2.9.1) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + net-http (0.6.0) + uri + netrc (0.11.0) + oj (3.15.1) + opensearch-ruby (3.4.0) + faraday (>= 1.0, < 3) + multi_json (>= 1.0) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + uri (1.0.2) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-opensearch (~> 1.1.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-opensearch/conf/fluent.conf b/docker-image/v1.18/debian-opensearch/conf/fluent.conf new file mode 100644 index 000000000..8cc30ff75 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/fluent.conf @@ -0,0 +1,42 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type opensearch + @id out_os + @log_level info + include_tag_key true + host "#{ENV['FLUENT_OPENSEARCH_HOST']}" + port "#{ENV['FLUENT_OPENSEARCH_PORT']}" + path "#{ENV['FLUENT_OPENSEARCH_PATH']}" + scheme "#{ENV['FLUENT_OPENSEARCH_SCHEME'] || 'http'}" + ssl_verify "#{ENV['FLUENT_OPENSEARCH_SSL_VERIFY'] || 'true'}" + ssl_version "#{ENV['FLUENT_OPENSEARCH_SSL_VERSION'] || 'TLSv1_2'}" + ca_file "#{ENV['FLUENT_OPENSEARCH_CA_PATH']}" + user "#{ENV['FLUENT_OPENSEARCH_USER']}" + password "#{ENV['FLUENT_OPENSEARCH_PASSWORD']}" + client_cert "#{ENV['FLUENT_OPENSEARCH_CLIENT_CERT']}" + client_key "#{ENV['FLUENT_OPENSEARCH_CLIENT_KEY']}" + client_key_pass "#{ENV['FLUENT_OPENSEARCH_CLIENT_KEY_PASS']}" + index_name "#{ENV['FLUENT_OPENSEARCH_INDEX_NAME'] || 'fluentd'}" + logstash_dateformat "#{ENV['FLUENT_OPENSEARCH_LOGSTASH_DATEFORMAT'] || '%Y.%m.%d'}" + logstash_format "#{ENV['FLUENT_OPENSEARCH_LOGSTASH_FORMAT'] || 'false'}" + logstash_prefix "#{ENV['FLUENT_OPENSEARCH_LOGSTASH_PREFIX'] || 'logstash'}" + logstash_prefix_separator "#{ENV['FLUENT_OPENSEARCH_LOGSTASH_PREFIX_SEPARATOR'] || '-'}" + <buffer> + flush_thread_count "#{ENV['FLUENT_OPENSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '1'}" + flush_mode "#{ENV['FLUENT_OPENSEARCH_BUFFER_FLUSH_MODE'] || 'interval'}" + flush_interval "#{ENV['FLUENT_OPENSEARCH_BUFFER_FLUSH_INTERVAL'] || '60s'}" + chunk_limit_size "#{ENV['FLUENT_OPENSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '8M'}" + total_limit_size "#{ENV['FLUENT_OPENSEARCH_BUFFER_TOTAL_LIMIT_SIZE'] || '512M'}" + retry_max_interval "#{ENV['FLUENT_OPENSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}" + retry_timeout "#{ENV['FLUENT_OPENSEARCH_BUFFER_RETRY_TIMEOUT'] || '72h'}" + retry_forever "#{ENV['FLUENT_OPENSEARCH_BUFFER_RETRY_FOREVER'] || 'false'}" + </buffer> +</match> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-opensearch/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/prometheus.conf b/docker-image/v1.18/debian-opensearch/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-opensearch/conf/systemd.conf b/docker-image/v1.18/debian-opensearch/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-opensearch/conf/tail_container_parse.conf b/docker-image/v1.18/debian-opensearch/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-opensearch/entrypoint.sh b/docker-image/v1.18/debian-opensearch/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-opensearch/plugins/.gitkeep b/docker-image/v1.18/debian-opensearch/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-opensearch/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-opensearch/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-opensearch/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-opensearch/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-opensearch/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-papertrail/.dockerignore b/docker-image/v1.18/debian-papertrail/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-papertrail/Dockerfile b/docker-image/v1.18/debian-papertrail/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-papertrail/Gemfile b/docker-image/v1.18/debian-papertrail/Gemfile new file mode 100644 index 000000000..c3cd2b0a3 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-papertrail", "~> 0.2.6" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-papertrail/Gemfile.lock b/docker-image/v1.18/debian-papertrail/Gemfile.lock new file mode 100644 index 000000000..963554a1d --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/Gemfile.lock @@ -0,0 +1,152 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-mixin-config-placeholders (0.4.0) + fluentd + uuidtools (>= 2.1.5) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-papertrail (0.2.8) + fluent-mixin-config-placeholders (~> 0.4.0) + fluentd (>= 0.10, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + uuidtools (2.2.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-papertrail (~> 0.2.6) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-papertrail/conf/fluent.conf b/docker-image/v1.18/debian-papertrail/conf/fluent.conf new file mode 100644 index 000000000..8b0cc403c --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/fluent.conf @@ -0,0 +1,26 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + + +## Capture audit logs +#<match kube-apiserver-audit> +# @type papertrail +# +# papertrail_host "#{ENV['FLUENT_PAPERTRAIL_AUDIT_HOST']}" +# papertrail_port "#{ENV['FLUENT_PAPERTRAIL_AUDIT_PORT']}" +#</match> + +<match **> + @type papertrail + @id out_papertrail + + papertrail_host "#{ENV['FLUENT_PAPERTRAIL_HOST']}" + papertrail_port "#{ENV['FLUENT_PAPERTRAIL_PORT']}" + +</match> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes.conf new file mode 100644 index 000000000..68fc59c5e --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes.conf @@ -0,0 +1,64 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + +<filter kube-apiserver-audit> + @type record_transformer + @id filter_rt_kube_apiserver_audit + enable_ruby true + <record> + hostname #{ENV['FLUENT_HOSTNAME']} + program kube-apiserver-audit + severity info + facility local0 + message ${record} + </record> +</filter> + +<filter kubernetes.**> + @type record_transformer + @id filter_rt_kube_logs + enable_ruby true + <record> + hostname #{ENV['FLUENT_HOSTNAME']}-${record["kubernetes"]["namespace_name"]}-${record["kubernetes"]["pod_name"]} + program ${record["kubernetes"]["container_name"]} + severity info + facility local0 + message ${record['log'] || record['message']} + </record> +</filter> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-papertrail/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/prometheus.conf b/docker-image/v1.18/debian-papertrail/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-papertrail/conf/systemd.conf b/docker-image/v1.18/debian-papertrail/conf/systemd.conf new file mode 100644 index 000000000..01ce25607 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/systemd.conf @@ -0,0 +1,58 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + +<filter bootkube> + @type record_transformer + @id filter_rt_bootkube + enable_ruby true + <record> + hostname #{ENV['FLUENT_HOSTNAME']} + program kube-bootkube + severity info + facility local0 + message ${record['log']} + </record> +</filter> diff --git a/docker-image/v1.18/debian-papertrail/conf/tail_container_parse.conf b/docker-image/v1.18/debian-papertrail/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-papertrail/entrypoint.sh b/docker-image/v1.18/debian-papertrail/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-papertrail/plugins/.gitkeep b/docker-image/v1.18/debian-papertrail/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-papertrail/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-papertrail/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-papertrail/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-papertrail/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-papertrail/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-s3/.dockerignore b/docker-image/v1.18/debian-s3/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-s3/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-s3/Dockerfile b/docker-image/v1.18/debian-s3/Dockerfile new file mode 100644 index 000000000..f59d2e80f --- /dev/null +++ b/docker-image/v1.18/debian-s3/Dockerfile @@ -0,0 +1,59 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev curl" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && curl -sL -o columnify_0.1.0_Linux_x86_64.tar.gz https://github.com/reproio/columnify/releases/download/v0.1.0/columnify_0.1.0_Linux_x86_64.tar.gz \ + && tar xf columnify_0.1.0_Linux_x86_64.tar.gz \ + && rm LICENSE README.md columnify_0.1.0_Linux_x86_64.tar.gz \ + && mv columnify /usr/local/bin/ \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-s3/Gemfile b/docker-image/v1.18/debian-s3/Gemfile new file mode 100644 index 000000000..e1a584542 --- /dev/null +++ b/docker-image/v1.18/debian-s3/Gemfile @@ -0,0 +1,22 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "aws-sdk-s3", "~> 1.101" +gem "fluent-plugin-s3", "~> 1.7.0" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-s3/Gemfile.lock b/docker-image/v1.18/debian-s3/Gemfile.lock new file mode 100644 index 000000000..5cb99d70e --- /dev/null +++ b/docker-image/v1.18/debian-s3/Gemfile.lock @@ -0,0 +1,170 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + aws-eventstream (1.3.0) + aws-partitions (1.1026.0) + aws-sdk-core (3.214.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.96.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sdk-s3 (1.176.1) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + aws-sdk-sqs (1.89.0) + aws-sdk-core (~> 3, >= 3.210.0) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.10.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-s3 (1.7.2) + aws-sdk-s3 (~> 1.60) + aws-sdk-sqs (~> 1.23) + fluentd (>= 0.14.22, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jmespath (1.6.2) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + aws-sdk-s3 (~> 1.101) + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-s3 (~> 1.7.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-s3/conf/fluent.conf b/docker-image/v1.18/debian-s3/conf/fluent.conf new file mode 100644 index 000000000..98dc47d3a --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/fluent.conf @@ -0,0 +1,31 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + # docs: https://docs.fluentd.org/v0.12/articles/out_s3 + # note: this configuration relies on the nodes have an IAM instance profile with access to your S3 bucket + @type s3 + @id out_s3 + @log_level info + s3_bucket "#{ENV['S3_BUCKET_NAME']}" + s3_region "#{ENV['S3_BUCKET_REGION']}" + s3_object_key_format "#{ENV['S3_OBJECT_KEY_FORMAT'] || '%{path}%Y/%m/%d/cluster-log-%{index}.%{file_extension}'}" + <inject> + time_key time + tag_key tag + localtime false + </inject> + <buffer> + @type file + path /var/log/fluentd-buffers/s3.buffer + timekey "#{ENV['S3_TIMEKEY'] || '3600'}" + timekey_use_utc true + chunk_limit_size "#{ENV['S3_CHUNK_LIMIT_SIZE'] || '256m'}" + </buffer> +</match> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes.conf b/docker-image/v1.18/debian-s3/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-s3/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-s3/conf/prometheus.conf b/docker-image/v1.18/debian-s3/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-s3/conf/systemd.conf b/docker-image/v1.18/debian-s3/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-s3/conf/tail_container_parse.conf b/docker-image/v1.18/debian-s3/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-s3/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-s3/entrypoint.sh b/docker-image/v1.18/debian-s3/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-s3/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-s3/plugins/.gitkeep b/docker-image/v1.18/debian-s3/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-s3/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-s3/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-s3/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-s3/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-s3/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-s3/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-syslog/.dockerignore b/docker-image/v1.18/debian-syslog/.dockerignore new file mode 100644 index 000000000..921ed3fe9 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.18/debian-syslog/Dockerfile b/docker-image/v1.18/debian-syslog/Dockerfile new file mode 100644 index 000000000..08ae5785a --- /dev/null +++ b/docker-image/v1.18/debian-syslog/Dockerfile @@ -0,0 +1,55 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.18.0-debian-amd64-1.0 + +LABEL maintainer="Eduardo Silva <eduardo@treasure-data.com>" +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/3.2.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/3.2.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ +RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev" \ + runtimeDeps="" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps $runtimeDeps net-tools \ + && gem install bundler --version 2.4.17 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + '*-dev' \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ +COPY ./conf/prometheus.conf /fluentd/etc/ +COPY ./conf/tail_container_parse.conf /fluentd/etc/ +COPY ./conf/kubernetes/*.conf /fluentd/etc/kubernetes/ +RUN touch /fluentd/etc/disable.conf + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" +# Override LD_PRELOAD in fluentd docker image +# Set "/usr/lib/libjemalloc.so.2" if you want to enable jemalloc explicitly +ENV LD_PRELOAD="" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.18/debian-syslog/Gemfile b/docker-image/v1.18/debian-syslog/Gemfile new file mode 100644 index 000000000..d7e4f38a4 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.18.0" +gem "oj", "~> 3.15.0" +gem "rexml", "~> 3.2.5" +gem "fluent-plugin-multi-format-parser", "~> 1.0.0" +gem "fluent-plugin-concat", "~> 2.5.0" +gem "fluent-plugin-grok-parser", "~> 2.6.2" +gem "fluent-plugin-prometheus", "~> 2.1.0" +gem 'fluent-plugin-json-in-json-2', ">= 1.0.2" +gem "fluent-plugin-record-modifier", "~> 2.1.0" +gem "fluent-plugin-detect-exceptions", "~> 0.0.13" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0" +gem "fluent-plugin-parser-cri", "~> 0.1.0" +gem "fluent-plugin-remote_syslog" +gem "fluent-plugin-kubernetes_metadata_filter", "~> 3.6.0" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.1.0" diff --git a/docker-image/v1.18/debian-syslog/Gemfile.lock b/docker-image/v1.18/debian-syslog/Gemfile.lock new file mode 100644 index 000000000..09a305f5d --- /dev/null +++ b/docker-image/v1.18/debian-syslog/Gemfile.lock @@ -0,0 +1,151 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) + concurrent-ruby (1.3.4) + cool.io (1.9.0) + csv (3.3.1) + domain_name (0.6.20240107) + drb (2.2.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-concat (2.5.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-detect-exceptions (0.0.16) + fluentd (>= 1.14.2) + fluent-plugin-grok-parser (2.6.2) + fluentd (>= 0.14.6, < 2) + fluent-plugin-json-in-json-2 (1.0.2) + fluentd (>= 0.14.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (3.6.0) + fluentd (>= 0.14.0, < 1.19) + kubeclient (>= 4.0.0, < 5.0.0) + lru_redux + fluent-plugin-multi-format-parser (1.0.0) + fluentd (>= 0.14.0, < 2) + fluent-plugin-parser-cri (0.1.1) + fluentd (>= 1) + fluent-plugin-prometheus (2.1.0) + fluentd (>= 1.9.1, < 2) + prometheus-client (>= 2.1.0) + fluent-plugin-record-modifier (2.1.1) + fluentd (>= 1.0, < 2) + fluent-plugin-remote_syslog (1.1.0) + fluentd + remote_syslog_sender (>= 1.1.1) + fluent-plugin-rewrite-tag-filter (2.4.0) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.1.0) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 2.0.0) + fluentd (1.18.0) + base64 (~> 0.2) + bundler + cool.io (>= 1.4.5, < 2.0.0) + csv (~> 3.2) + drb (~> 2.2) + http_parser.rb (>= 0.5.1, < 0.9.0) + logger (~> 1.6) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.3.2, < 3.0.0) + sigdump (~> 0.2.5) + strptime (>= 0.2.4, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (~> 1.4) + yajl-ruby (~> 1.0) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-accept (1.7.0) + http-cookie (1.0.8) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + jsonpath (1.1.5) + multi_json + kubeclient (4.12.0) + http (>= 3.0, < 6.0) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) + rest-client (~> 2.0) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + logger (1.6.4) + lru_redux (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1203) + msgpack (1.7.5) + multi_json (1.15.0) + netrc (0.11.0) + oj (3.15.1) + ostruct (0.6.1) + prometheus-client (4.2.3) + base64 + public_suffix (6.0.1) + rake (13.2.1) + recursive-open-struct (1.3.1) + ostruct + remote_syslog_sender (1.2.2) + syslog_protocol + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rexml (3.2.9) + strscan + serverengine (2.4.0) + base64 (~> 0.1) + logger (~> 1.4) + sigdump (~> 0.2.2) + sigdump (0.2.5) + strptime (0.2.5) + strscan (3.1.2) + syslog_protocol (0.9.2) + systemd-journal (2.0.0) + ffi (~> 1.9) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.2) + tzinfo (>= 1.0.0) + webrick (1.9.1) + yajl-ruby (1.4.3) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-concat (~> 2.5.0) + fluent-plugin-detect-exceptions (~> 0.0.13) + fluent-plugin-grok-parser (~> 2.6.2) + fluent-plugin-json-in-json-2 (>= 1.0.2) + fluent-plugin-kubernetes_metadata_filter (~> 3.6.0) + fluent-plugin-multi-format-parser (~> 1.0.0) + fluent-plugin-parser-cri (~> 0.1.0) + fluent-plugin-prometheus (~> 2.1.0) + fluent-plugin-record-modifier (~> 2.1.0) + fluent-plugin-remote_syslog + fluent-plugin-rewrite-tag-filter (~> 2.4.0) + fluent-plugin-systemd (~> 1.1.0) + fluentd (= 1.18.0) + oj (~> 3.15.0) + rexml (~> 3.2.5) + +BUNDLED WITH + 2.4.19 diff --git a/docker-image/v1.18/debian-syslog/conf/fluent.conf b/docker-image/v1.18/debian-syslog/conf/fluent.conf new file mode 100644 index 000000000..941d44f57 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/fluent.conf @@ -0,0 +1,33 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" +@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" +@include kubernetes.conf +@include conf.d/*.conf + +<match **> + @type remote_syslog + @id out_kube_remote_syslog + host "#{ENV['SYSLOG_HOST']}" + port "#{ENV['SYSLOG_PORT']}" + severity debug + program fluentd + hostname ${kubernetes_host} + + protocol "#{ENV['SYSLOG_PROTOCOL'] || 'tcp'}" + tls "#{ENV['SYSLOG_TLS'] || 'false'}" + ca_file "#{ENV['SYSLOG_CA_FILE'] || ''}" + verify_mode "#{ENV['SYSLOG_VERIFY_MODE'] || ''}" + packet_size 65535 + + <buffer kubernetes_host> + flush_interval "#{ENV['FLUENT_SYSLOG_FLUSH_INTERVAL'] || use_default}" + </buffer> + + <format> + @type ltsv + </format> +</match> + diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes.conf new file mode 100644 index 000000000..bbdd453d6 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes.conf @@ -0,0 +1,39 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + +<label @FLUENT_LOG> + <match fluent.**> + @type null + @id ignore_fluent_logs + </match> +</label> + +@include kubernetes/cluster-autoscaler.conf +@include kubernetes/containers.conf +@include kubernetes/docker.conf +@include kubernetes/etcd.conf +@include kubernetes/glbc.conf +@include kubernetes/kube-apiserver-audit.conf +@include kubernetes/kube-apiserver.conf +@include kubernetes/kube-controller-manager.conf +@include kubernetes/kube-proxy.conf +@include kubernetes/kube-scheduler.conf +@include kubernetes/kubelet.conf +@include kubernetes/rescheduler.conf +@include kubernetes/salt.conf +@include kubernetes/startupscript.conf + + +<filter kubernetes.**> + @type kubernetes_metadata + @id filter_kube_metadata + kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || (host = ENV.fetch('KUBERNETES_SERVICE_HOST'); port = ENV.fetch('KUBERNETES_SERVICE_PORT'); host = (IPAddr.new(host).ipv6? ? '[' + host + ']' : host rescue host); 'https://' + host + ':' + port + '/api')}" + verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}" + ca_file "#{ENV['KUBERNETES_CA_FILE']}" + skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" + skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" + skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" + skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" + watch "#{ENV['FLUENT_KUBERNETES_WATCH'] || 'true'}" +</filter> + diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/cluster-autoscaler.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/cluster-autoscaler.conf new file mode 100644 index 000000000..5e045b017 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/cluster-autoscaler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-cluster-autoscaler.log.pos')}" + tag cluster-autoscaler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/containers.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/containers.conf new file mode 100644 index 000000000..1ea4d1ea4 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/containers.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_container_logs + path "#{ENV['FLUENT_CONTAINER_TAIL_PATH'] || '/var/log/containers/*.log'}" + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-containers.log.pos')}" + tag "#{ENV['FLUENT_CONTAINER_TAIL_TAG'] || 'kubernetes.*'}" + exclude_path "#{ENV['FLUENT_CONTAINER_TAIL_EXCLUDE_PATH'] || use_default}" + read_from_head true + @include ../tail_container_parse.conf +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/docker.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/docker.conf new file mode 100644 index 000000000..f79e010d4 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/docker.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-docker.log.pos')}" + tag docker + <parse> + @type regexp + expression /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/etcd.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/etcd.conf new file mode 100644 index 000000000..be565daef --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/etcd.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-etcd.log.pos')}" + tag etcd + <parse> + @type none + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/glbc.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/glbc.conf new file mode 100644 index 000000000..4010f4f1d --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/glbc.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-glbc.log.pos')}" + tag glbc + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-apiserver-audit.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-apiserver-audit.conf new file mode 100644 index 000000000..b48743bd7 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-apiserver-audit.conf @@ -0,0 +1,23 @@ +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" +<source> + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'kube-apiserver-audit.log.pos')}" + tag kube-apiserver-audit + <parse> + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/ + time_format %Y-%m-%dT%T.%L%Z + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-apiserver.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-apiserver.conf new file mode 100644 index 000000000..4e4bfad9e --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-apiserver.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-apiserver.log.pos')}" + tag kube-apiserver + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-controller-manager.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-controller-manager.conf new file mode 100644 index 000000000..c35f7edd7 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-controller-manager.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-controller-manager.log.pos')}" + tag kube-controller-manager + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-proxy.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-proxy.conf new file mode 100644 index 000000000..2ae19a287 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-proxy.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-proxy.log.pos')}" + tag kube-proxy + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-scheduler.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-scheduler.conf new file mode 100644 index 000000000..53ccdda55 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/kube-scheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kube-scheduler.log.pos')}" + tag kube-scheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/kubelet.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/kubelet.conf new file mode 100644 index 000000000..5a1e9eef5 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/kubelet.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-kubelet.log.pos')}" + tag kubelet + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/rescheduler.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/rescheduler.conf new file mode 100644 index 000000000..69d72a237 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/rescheduler.conf @@ -0,0 +1,11 @@ +<source> + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-rescheduler.log.pos')}" + tag rescheduler + <parse> + @type kubernetes + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/salt.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/salt.conf new file mode 100644 index 000000000..60b6b2b16 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/salt.conf @@ -0,0 +1,13 @@ + +<source> + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_EXTRA_DIR', ''), 'fluentd-salt.pos')}" + tag salt + <parse> + @type regexp + expression /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ + time_format %Y-%m-%d %H:%M:%S + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/kubernetes/startupscript.conf b/docker-image/v1.18/debian-syslog/conf/kubernetes/startupscript.conf new file mode 100644 index 000000000..e44bb75a2 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/kubernetes/startupscript.conf @@ -0,0 +1,10 @@ +<source> + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file "#{File.join('/var/log/', ENV.fetch('FLUENT_POS_DIR', ''), 'fluentd-startupscript.log.pos')}" + tag startupscript + <parse> + @type syslog + </parse> +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/prometheus.conf b/docker-image/v1.18/debian-syslog/conf/prometheus.conf new file mode 100644 index 000000000..37599dd8a --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/prometheus.conf @@ -0,0 +1,16 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/prometheus.conf.erb + +# Prometheus metric exposed on 0.0.0.0:24231/metrics +<source> + @type prometheus + @id in_prometheus + bind "#{ENV['FLUENTD_PROMETHEUS_BIND'] || '0.0.0.0'}" + port "#{ENV['FLUENTD_PROMETHEUS_PORT'] || '24231'}" + metrics_path "#{ENV['FLUENTD_PROMETHEUS_PATH'] || '/metrics'}" +</source> + +<source> + @type prometheus_output_monitor + @id in_prometheus_output_monitor +</source> diff --git a/docker-image/v1.18/debian-syslog/conf/systemd.conf b/docker-image/v1.18/debian-syslog/conf/systemd.conf new file mode 100644 index 000000000..cfc73eba8 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + </storage> + read_from_head true + tag kubelet +</source> + +# Logs from docker-systemd +<source> + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + </storage> + read_from_head true + tag docker.systemd +</source> + +# Logs from systemd-journal for interesting services. +<source> + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + <storage> + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + </storage> + read_from_head true + tag bootkube +</source> + diff --git a/docker-image/v1.18/debian-syslog/conf/tail_container_parse.conf b/docker-image/v1.18/debian-syslog/conf/tail_container_parse.conf new file mode 100644 index 000000000..873c6e5dc --- /dev/null +++ b/docker-image/v1.18/debian-syslog/conf/tail_container_parse.conf @@ -0,0 +1,4 @@ +<parse> + @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" + time_format "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TIME_FORMAT'] || '%Y-%m-%dT%H:%M:%S.%NZ'}" +</parse> diff --git a/docker-image/v1.18/debian-syslog/entrypoint.sh b/docker-image/v1.18/debian-syslog/entrypoint.sh new file mode 100755 index 000000000..a09f4369f --- /dev/null +++ b/docker-image/v1.18/debian-syslog/entrypoint.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/entrypoint.sh.erb + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.18/debian-syslog/plugins/.gitkeep b/docker-image/v1.18/debian-syslog/plugins/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/docker-image/v1.18/debian-syslog/plugins/parser_kubernetes.rb b/docker-image/v1.18/debian-syslog/plugins/parser_kubernetes.rb new file mode 100644 index 000000000..0f593caaf --- /dev/null +++ b/docker-image/v1.18/debian-syslog/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/m} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['expression'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end diff --git a/docker-image/v1.18/debian-syslog/plugins/parser_multiline_kubernetes.rb b/docker-image/v1.18/debian-syslog/plugins/parser_multiline_kubernetes.rb new file mode 100644 index 000000000..3b94ee282 --- /dev/null +++ b/docker-image/v1.18/debian-syslog/plugins/parser_multiline_kubernetes.rb @@ -0,0 +1,69 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# <source> +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# <parse> +# @type multiline_kubernetes +# </parse> +# </source> +# +# ---- EOF --- + +require 'fluent/plugin/parser_multiline' + +module Fluent + module Plugin + class MultilineKubernetesParser < MultilineParser + Fluent::Plugin.register_parser("multiline_kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/} + CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N" + + def configure(conf) + conf['format_firstline'] = CONF_FORMAT_FIRSTLINE + conf['format1'] = CONF_FORMAT1 + conf['time_format'] = CONF_TIME_FORMAT + super + end + end + end +end