Merge branch 'fluent:master' into fix/timeout-issue

fluent · Oct 29, 2024 · 37a4e5e · 37a4e5e
2 parents 53123b8 + 6fccaa2
commit 37a4e5e
Show file tree

Hide file tree

Showing 41 changed files with 3,348 additions and 838 deletions.
diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml
@@ -42,6 +42,8 @@ jobs:
           - "-DFLB_COVERAGE=On"
           - "-DFLB_SANITIZE_MEMORY=On"
           - "-DFLB_SANITIZE_THREAD=On"
+          - "-DFLB_SIMD=On"
+          - "-DFLB_SIMD=Off"
         compiler:
           - gcc
           - clang

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -141,6 +141,7 @@ option(FLB_SIGNV4              "Enable AWS Signv4 support"    Yes)
 option(FLB_AWS                 "Enable AWS support"           Yes)
 option(FLB_STATIC_CONF         "Build binary using static configuration")
 option(FLB_STREAM_PROCESSOR    "Enable Stream Processor"                    Yes)
+option(FLB_SIMD                "Enable SIMD support"                         No)
 option(FLB_CORO_STACK_SIZE     "Set coroutine stack size")
 option(FLB_AVRO_ENCODER        "Build with Avro encoding support"            No)
 option(FLB_AWS_ERROR_REPORTER  "Build with aws error reporting support"      No)
@@ -181,6 +182,10 @@ option(FLB_EVENT_LOOP_KQUEUE    "Enable kqueue(2) event loop backend" No)
 option(FLB_EVENT_LOOP_SELECT    "Enable select(2) event loop backend" No)
 option(FLB_EVENT_LOOP_LIBEVENT  "Enable libevent event loop backend"  No)
 
+# SIMD support
+if(FLB_SIMD)
+  FLB_DEFINITION(FLB_HAVE_SIMD)
+endif()
 
 if(DEFINED FLB_NIGHTLY_BUILD AND NOT "${FLB_NIGHTLY_BUILD}" STREQUAL "")
   FLB_DEFINITION_VAL(FLB_NIGHTLY_BUILD ${FLB_NIGHTLY_BUILD})

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -711,7 +711,8 @@ The dependencies must be present:
 
 * Microsoft Visual Studio C/C++ toolchain. The CI automation uses MSVC 2019 at time of writing. MSVC Community Edition works fine.
 * [CMake](https://cmake.org/) 3.x on the `PATH`
-* A build of [OpenSSL](https://www.openssl.org/) as static libraries, pointed to by the `-DOPENSSL_ROOT_DIR` CMake variable. The CI automation uses [Chocolatey](https://chocolatey.org/) to `choco install -y openssl`.
+* A build of [OpenSSL](https://www.openssl.org/) as static libraries, pointed to by the `-DOPENSSL_ROOT_DIR` CMake variable.
+* The CI automation uses vcpkg to install dependencies [](https://github.com/fluent/fluent-bit/blob/master/.github/workflows/call-build-windows.yaml#L148)
 * `flex.exe` and `bison.exe` must be present on the `PATH`. The CI automation uses https://github.com/lexxmark/winflexbison.
 
 Assuming that `cmake` is on the `PATH`, Visual Studio is installed,
@@ -731,6 +732,134 @@ The build output will be `bin\Debug\fluent-bit.exe`.
 
 If in doubt, check the CI and build automation files referenced above for specifics.
 
+### Building on a Windows Server 2022
+
+The following steps have been tested on a Windows Server 2022 Datacenter edition on top of GCP.
+
+1. **Download and Install Visual Studio 2022** (Community Edition)
+    - **Download**: Go to [Visual Studio Download Page](https://visualstudio.microsoft.com/downloads/).
+    - **Install**:
+        - Select **Community Edition** and check the following components during installation:
+            - **Desktop development with C++**
+            - **Linux development with C++**
+
+2. **Install Flex and Bison**
+    1. Create a new file called `setup-flex-bison.ps1` and paste the following script:
+
+    ```powershell
+    # Define variables for Flex and Bison
+    $flexBisonUrl = "https://sourceforge.net/projects/winflexbison/files/win_flex_bison3-latest.zip/download"
+    $downloadPath = "$env:TEMP\win_flex_bison.zip"
+    $extractPath = "C:\win_flex_bison"
+    $flexExe = "flex.exe"
+    $bisonExe = "bison.exe"
+
+    # Step 2: Download and Setup Flex and Bison
+    Write-Output "Downloading win_flex_bison..."
+    Invoke-WebRequest -Uri $flexBisonUrl -OutFile $downloadPath
+
+    # Create the extract directory if it does not exist
+    If (!(Test-Path -Path $extractPath)) {
+        New-Item -ItemType Directory -Path $extractPath
+    }
+
+    # Extract the zip file
+    Write-Output "Extracting win_flex_bison..."
+    Add-Type -AssemblyName System.IO.Compression.FileSystem
+    [System.IO.Compression.ZipFile]::ExtractToDirectory($downloadPath, $extractPath)
+
+    # Rename the executables
+    Write-Output "Renaming executables..."
+    Rename-Item "$extractPath\win_flex.exe" "$extractPath\$flexExe" -Force
+    Rename-Item "$extractPath\win_bison.exe" "$extractPath\$bisonExe" -Force
+
+    # Add Flex and Bison path to system environment variables
+    Write-Output "Adding Flex and Bison path to environment variables..."
+    $envPath = [System.Environment]::GetEnvironmentVariable("Path", "Machine")
+    If ($envPath -notlike "*$extractPath*") {
+        [System.Environment]::SetEnvironmentVariable("Path", "$envPath;$extractPath", "Machine")
+        Write-Output "Path updated. Please restart your command prompt to apply changes."
+    } else {
+        Write-Output "Path already contains the Flex and Bison directory."
+    }
+
+    # Cleanup
+    Remove-Item $downloadPath
+
+    Write-Output "Flex and Bison setup complete."
+    ```
+
+    2. Run the Script: Open PowerShell as administrator.
+
+    ```powershell
+    .\setup-flex-bison.ps1
+    ```
+
+    3. Restart the command prompt: After the script completes, restart your command prompt or Visual Studio for the changes to take effect.
+
+3. **Create `vcpkg.json` file for the dependencies**
+
+    In the root of your project, create a `vcpkg.json` file with the following content:
+
+    ```json
+    {
+        "name": "fluent-bit",
+        "version": "3.2.0",
+        "dependencies": [
+            {
+                "name": "openssl",
+                "default-features": false
+            },
+            {
+                "name": "libyaml",
+                "default-features": false
+            }
+        ],
+        "builtin-baseline": "9f5925e81bbcd9c8c34cc7a8bd25e3c557b582b2"
+    }
+    ```
+
+4. **Install dependencies using `vcpkg`**
+
+    ```bash
+    vcpkg install --triplet x64-windows-static
+    ```
+
+    You should see output like:
+
+    ```bash
+    libyaml:x64-windows-static      0.2.5#5      A C library for parsing and emitting YAML.
+    openssl:x64-windows-static      3.3.2#1      OpenSSL is an open source project that provides SSL and TLS.
+    ```
+
+5. **Link `vcpkg` with Visual Studio**
+
+    ```bash
+    vcpkg integrate install
+    ```
+
+6. **Generate the Visual Studio solution of Fluent Bit using CMake**
+
+    ```bash
+    cd build
+    cmake -G "Visual Studio 17 2022" -DFLB_TESTS_INTERNAL=Off -DFLB_TESTS_RUNTIME=Off -DCMAKE_TOOLCHAIN_FILE="C:/Program Files/Microsoft Visual Studio/2022/Community/VC/vcpkg/scripts/buildsystems/vcpkg.cmake" -DOPENSSL_ROOT_DIR=C:/path/to/your/vcpkg_installed/x64-windows-static -DFLB_LIBYAML_DIR=C:/path/to/your/vcpkg_installed/x64-windows-static ..
+    ```
+
+    **Notes**:
+    - Replace `C:/path/to/your/vcpkg_installed/x64-windows-static` with the actual path where `vcpkg` installed OpenSSL and LibYAML.
+    - When installing with `vcpkg`, you can also specify a different install root using `--x-install-root`.
+    - This will generate a Visual Studio solution file, which you can open and compile.
+
+7. **Run the binary build**
+
+    ```bash
+    cmake --build . --parallel 4 --clean-first
+    ```
+
+    **Notes**:
+    - You can choose to omit the `--parallel` option.
+    - The `--clean-first` option will clear cache and start a fresh clean build.
+
 ### Valgrind
 
 [Valgrind](https://valgrind.org/) is a tool that will help you detect and diagnose memory issues in your code. It will check for memory leaks and invalid memory accesses.

diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h
@@ -55,7 +55,10 @@ enum cf_file_format {
 enum section_type {
     FLB_CF_SERVICE = 0,           /* [SERVICE]           */
     FLB_CF_PARSER,                /* [PARSER]            */
-    FLB_CF_MULTILINE_PARSER,      /* [MULTILINE_PARSER]  */
+    FLB_CF_MULTILINE_PARSER,      /* multiline_parser    */
+    FLB_CF_STREAM_PROCESSOR,      /* stream_processor    */
+    FLB_CF_PLUGINS,               /* plugins             */
+    FLB_CF_UPSTREAM_SERVERS,      /* upstream_servers    */
     FLB_CF_CUSTOM,                /* [CUSTOM]            */
     FLB_CF_INPUT,                 /* [INPUT]             */
     FLB_CF_FILTER,                /* [FILTER]            */
@@ -97,7 +100,16 @@ struct flb_cf {
     struct mk_list parsers;
     struct mk_list multiline_parsers;
 
-    /* custom plugins */
+    /* stream processor: every entry is added as a task */
+    struct mk_list stream_processors;
+
+    /* external plugins (.so) */
+    struct mk_list plugins;
+
+    /* upstream servers */
+    struct mk_list upstream_servers;
+
+    /* 'custom' type plugins */
     struct mk_list customs;
 
     /* pipeline */

diff --git a/include/fluent-bit/flb_parser.h b/include/fluent-bit/flb_parser.h
@@ -75,7 +75,7 @@ enum {
     FLB_PARSER_TYPE_HEX,
 };
 
-static inline time_t flb_parser_tm2time(const struct flb_tm *src, 
+static inline time_t flb_parser_tm2time(const struct flb_tm *src,
                                         int use_system_timezone)
 {
     struct tm tmp;
@@ -107,6 +107,11 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
                                      struct flb_config *config);
 int flb_parser_conf_file_stat(const char *file, struct flb_config *config);
 int flb_parser_conf_file(const char *file, struct flb_config *config);
+int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf,
+                                       struct flb_config *config);
+int flb_parser_load_multiline_parser_definitions(const char *cfg, struct flb_cf *cf,
+                                                 struct flb_config *config);
+
 void flb_parser_destroy(struct flb_parser *parser);
 struct flb_parser *flb_parser_get(const char *name, struct flb_config *config);
 int flb_parser_do(struct flb_parser *parser, const char *buf, size_t length,

diff --git a/include/fluent-bit/flb_plugin.h b/include/fluent-bit/flb_plugin.h
@@ -47,8 +47,12 @@ struct flb_plugins {
 struct flb_plugins *flb_plugin_create();
 int flb_plugin_load(char *path, struct flb_plugins *ctx,
                     struct flb_config *config);
+
 int flb_plugin_load_router(char *path, struct flb_config *config);
+
 int flb_plugin_load_config_file(const char *file, struct flb_config *config);
+int flb_plugin_load_config_format(struct flb_cf *cf, struct flb_config *config);
+
 void flb_plugin_destroy(struct flb_plugins *ctx);
 
 #endif