-
-
Notifications
You must be signed in to change notification settings - Fork 14.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fluent-bit: link against Nix dependencies, fix Darwin builds, and add…
… NixOS module
- Loading branch information
1 parent
5f02383
commit 82b291a
Showing
8 changed files
with
262 additions
and
61 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| { | ||
| config, | ||
| lib, | ||
| pkgs, | ||
| ... | ||
| }: | ||
| let | ||
| cfg = config.services.fluent-bit; | ||
|
|
||
| yamlFormat = pkgs.formats.yaml { }; | ||
|
|
||
| configurationFile = | ||
| if (cfg.configurationFile == null) then | ||
| (yamlFormat.generate "fluent-bit.yaml" cfg.configuration) | ||
| else | ||
| cfg.configurationFile; | ||
| in | ||
| { | ||
| options.services.fluent-bit = { | ||
| enable = lib.mkEnableOption "Fluent Bit"; | ||
| package = lib.mkPackageOption pkgs "fluent-bit" { }; | ||
| configurationFile = lib.mkOption { | ||
| type = lib.types.nullOr lib.types.path; | ||
| default = null; | ||
| description = '' | ||
| Fluent Bit configuration. See | ||
| <https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml> | ||
| for supported values. | ||
| {option}`configurationFile` takes precedence over {option}`configuration`. | ||
| Note: Restricted evaluation blocks access to paths outside the Nix store. | ||
| This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done. | ||
| As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change. | ||
| `systemctl restart fluent-bit.service` must be used instead. | ||
| ''; | ||
| example = "/etc/fluent-bit/fluent-bit.yaml"; | ||
| }; | ||
| configuration = lib.mkOption { | ||
| type = yamlFormat.type; | ||
| default = { }; | ||
| description = '' | ||
| See {option}`configurationFile`. | ||
| {option}`configurationFile` takes precedence over {option}`configuration`. | ||
| ''; | ||
| example = { | ||
| service = { | ||
| grace = 30; | ||
| }; | ||
| pipeline = { | ||
| inputs = [ | ||
| { | ||
| name = "systemd"; | ||
| systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; | ||
| } | ||
| ]; | ||
| outputs = [ | ||
| { | ||
| name = "file"; | ||
| path = "/var/log/fluent-bit"; | ||
| file = "fluent-bit.out"; | ||
| } | ||
| ]; | ||
| }; | ||
| }; | ||
| }; | ||
| # See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section. | ||
| graceLimit = lib.mkOption { | ||
| type = lib.types.nullOr ( | ||
| lib.types.oneOf [ | ||
| lib.types.ints.positive | ||
| lib.types.str | ||
| ] | ||
| ); | ||
| default = null; | ||
| description = '' | ||
| The grace time limit. Sets the systemd unit's `TimeoutStopSec`. | ||
| The `service.grace` option in the Fluent Bit configuration should be ≤ this option. | ||
| ''; | ||
| example = 30; | ||
| }; | ||
| }; | ||
|
|
||
| config = lib.mkIf cfg.enable { | ||
| # See https://github.com/fluent/fluent-bit/blob/v3.2.3/init/systemd.in. | ||
| systemd.services.fluent-bit = { | ||
| description = "Fluent Bit"; | ||
| after = [ "network.target" ]; | ||
| requires = [ "network.target" ]; | ||
| wantedBy = [ "multi-user.target" ]; | ||
| serviceConfig = | ||
| { | ||
| DynamicUser = true; | ||
| # See https://nixos.org/manual/nixos/stable#sec-logging. | ||
| SupplementaryGroups = "systemd-journal"; | ||
| ExecStart = builtins.concatStringsSep " " [ | ||
| "${cfg.package}/bin/fluent-bit" | ||
| "--config ${configurationFile}" | ||
| ]; | ||
| Restart = "always"; | ||
| } | ||
| // (lib.optionalAttrs (cfg.graceLimit != null) { | ||
| TimeoutStopSec = cfg.graceLimit; | ||
| }); | ||
| }; | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| import ./make-test-python.nix ( | ||
| { lib, pkgs, ... }: | ||
| { | ||
| name = "fluent-bit"; | ||
|
|
||
| nodes.machine = | ||
| { config, pkgs, ... }: | ||
| { | ||
| services.fluent-bit = { | ||
| enable = true; | ||
| configuration = { | ||
| pipeline = { | ||
| inputs = [ | ||
| { | ||
| name = "systemd"; | ||
| systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; | ||
| } | ||
| ]; | ||
| outputs = [ | ||
| { | ||
| name = "file"; | ||
| path = "/var/log/fluent-bit"; | ||
| file = "fluent-bit.out"; | ||
| } | ||
| ]; | ||
| }; | ||
| }; | ||
| }; | ||
|
|
||
| systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit"; | ||
| }; | ||
|
|
||
| testScript = '' | ||
| start_all() | ||
| machine.wait_for_unit("fluent-bit.service") | ||
| machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out") | ||
| ''; | ||
| } | ||
| ) |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,83 +1,145 @@ | ||
| { | ||
| lib, | ||
| stdenv, | ||
| fetchFromGitHub, | ||
| bison, | ||
| c-ares, | ||
| cmake, | ||
| curl, | ||
| fetchFromGitHub, | ||
| flex, | ||
| bison, | ||
| systemd, | ||
| postgresql, | ||
| openssl, | ||
| jemalloc, | ||
| libbacktrace, | ||
| libbpf, | ||
| libnghttp2, | ||
| libyaml, | ||
| darwin, | ||
| luajit, | ||
| nix-update-script, | ||
| nixosTests, | ||
| openssl, | ||
| pkg-config, | ||
| postgresql, | ||
| rdkafka, | ||
| stdenv, | ||
| systemd, | ||
| versionCheckHook, | ||
| zlib, | ||
| zstd, | ||
| }: | ||
|
|
||
| stdenv.mkDerivation (finalAttrs: { | ||
| stdenv.mkDerivation rec { | ||
| pname = "fluent-bit"; | ||
| version = "3.2.3"; | ||
|
|
||
| src = fetchFromGitHub { | ||
| owner = "fluent"; | ||
| repo = "fluent-bit"; | ||
| rev = "v${finalAttrs.version}"; | ||
| tag = "v${version}"; | ||
| hash = "sha256-5Oyw3nHlAyywF+G0UiGyi1v+jAr8eyKt/1cDT5FdJXQ="; | ||
| }; | ||
|
|
||
| # optional only to avoid linux rebuild | ||
| patches = lib.optionals stdenv.hostPlatform.isDarwin [ ./macos-11-sdk-compat.patch ]; | ||
| # `src/CMakeLists.txt` installs fluent-bit's systemd unit files at the path in the `SYSTEMD_UNITDIR` CMake variable. | ||
| # | ||
| # The initial value of `SYSTEMD_UNITDIR` is set in `cmake/FindJournald` which uses pkg-config to find the systemd | ||
| # unit directory. `src/CMakeLists.txt` only sets `SYSTEMD_UNITDIR` to `/lib/systemd/system` if it's unset. | ||
| # | ||
| # Unfortunately, this resolves to systemd's Nix store path which is immutable. Consequently, CMake fails when trying | ||
| # to install fluent-bit's systemd unit files to the systemd Nix store path. | ||
| # | ||
| # We fix this by replacing `${SYSTEMD_UNITDIR}` instances in `src/CMakeLists.txt`. | ||
| postPatch = '' | ||
| substituteInPlace src/CMakeLists.txt \ | ||
| --replace-fail \''${SYSTEMD_UNITDIR} $out/lib/systemd/system | ||
| ''; | ||
|
|
||
| # The source build documentation covers some dependencies and CMake options. | ||
| # | ||
| # - Linux: https://docs.fluentbit.io/manual/installation/sources/build-and-install | ||
| # - Darwin: https://docs.fluentbit.io/manual/installation/macos#compile-from-source | ||
| # | ||
| # Unfortunately, fluent-bit vends many dependencies (e.g. luajit) as source files and tries to compile them by | ||
| # default, with none of their dependencies and CMake options documented. | ||
| # | ||
| # Fortunately, there's the undocumented `FLB_PREFER_SYSTEM_LIBS` CMake option to link against system libraries for | ||
| # some dependencies. | ||
| # | ||
| # See https://github.com/fluent/fluent-bit/blob/v3.2.3/CMakeLists.txt#L211-L218. | ||
| # | ||
| # Like `FLB_PREFER_SYSTEM_LIBS`, several CMake options aren't documented. | ||
| # | ||
| # See https://github.com/fluent/fluent-bit/blob/v3.2.3/CMakeLists.txt#L111-L157. | ||
| # | ||
| # The CMake options may differ across target platforms. We'll stick to the minimum. | ||
| # | ||
| # See https://github.com/fluent/fluent-bit/tree/v3.2.3/packaging/distros. | ||
|
|
||
| strictDeps = true; | ||
|
|
||
| nativeBuildInputs = [ | ||
| bison | ||
| cmake | ||
| flex | ||
| bison | ||
| pkg-config | ||
| ]; | ||
|
|
||
| buildInputs = | ||
| [ | ||
| openssl | ||
| c-ares | ||
| # Needed by rdkafka. | ||
| curl | ||
| jemalloc | ||
| libbacktrace | ||
| libnghttp2 | ||
| libyaml | ||
| luajit | ||
| openssl | ||
| postgresql | ||
| rdkafka | ||
| # Needed by rdkafka. | ||
| zlib | ||
| # Needed by rdkafka. | ||
| zstd | ||
| ] | ||
| ++ lib.optionals stdenv.hostPlatform.isLinux [ systemd ] | ||
| ++ lib.optionals stdenv.hostPlatform.isDarwin [ | ||
| darwin.apple_sdk_11_0.frameworks.IOKit | ||
| darwin.apple_sdk_11_0.frameworks.Foundation | ||
| ++ lib.optionals stdenv.hostPlatform.isLinux [ | ||
| # libbpf doesn't build for Darwin yet. | ||
| libbpf | ||
| systemd | ||
| ]; | ||
|
|
||
| cmakeFlags = [ | ||
| "-DFLB_RELEASE=ON" | ||
| "-DFLB_METRICS=ON" | ||
| "-DFLB_HTTP_SERVER=ON" | ||
| "-DFLB_OUT_PGSQL=ON" | ||
| ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" ]; | ||
|
|
||
| env.NIX_CFLAGS_COMPILE = toString ( | ||
| # Used by the embedded luajit, but is not predefined on older mac SDKs. | ||
| lib.optionals stdenv.hostPlatform.isDarwin [ "-DTARGET_OS_IPHONE=0" ] | ||
| # Assumes GNU version of strerror_r, and the posix version has an | ||
| # incompatible return type. | ||
| ++ lib.optionals (!stdenv.hostPlatform.isGnu) [ "-Wno-int-conversion" ] | ||
| ); | ||
| cmakeFlags = | ||
| [ | ||
| "-DFLB_RELEASE=Yes" | ||
| "-DFLB_PREFER_SYSTEM_LIBS=Yes" | ||
| ] | ||
| ++ lib.optionals stdenv.cc.isClang [ | ||
| # FLB_SECURITY causes bad linker options for Clang to be set. | ||
| "-DFLB_SECURITY=Off" | ||
| ]; | ||
|
|
||
| outputs = [ | ||
| "out" | ||
| "dev" | ||
| ]; | ||
|
|
||
| postPatch = '' | ||
| substituteInPlace src/CMakeLists.txt \ | ||
| --replace /lib/systemd $out/lib/systemd | ||
| ''; | ||
| doInstallCheck = true; | ||
|
|
||
| nativeInstallCheckInputs = [ versionCheckHook ]; | ||
|
|
||
| versionCheckProgram = "${builtins.placeholder "out"}/bin/${meta.mainProgram}"; | ||
|
|
||
| versionCheckProgramArg = "--version"; | ||
|
|
||
| passthru = { | ||
| tests = lib.optionalAttrs stdenv.isLinux { | ||
| inherit (nixosTests) fluent-bit; | ||
| }; | ||
|
|
||
| updateScript = nix-update-script { }; | ||
| }; | ||
|
|
||
| meta = { | ||
| changelog = "https://github.com/fluent/fluent-bit/releases/tag/v${finalAttrs.version}"; | ||
| description = "Log forwarder and processor, part of Fluentd ecosystem"; | ||
| description = "Fast and lightweight logs and metrics processor for Linux, BSD, OSX and Windows"; | ||
| homepage = "https://fluentbit.io"; | ||
| license = lib.licenses.asl20; | ||
| maintainers = with lib.maintainers; [ | ||
| samrose | ||
| fpletz | ||
| ]; | ||
| platforms = lib.platforms.unix; | ||
| mainProgram = "fluent-bit"; | ||
| maintainers = with lib.maintainers; [ samrose ]; | ||
| }; | ||
| }) | ||
| } |