diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 0fa50f11890..36d1f85d841 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -42,6 +42,8 @@ jobs: - "-DFLB_COVERAGE=On" - "-DFLB_SANITIZE_MEMORY=On" - "-DFLB_SANITIZE_THREAD=On" + - "-DFLB_SIMD=On" + - "-DFLB_SIMD=Off" compiler: - gcc - clang diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e6c5ac2613..eff73766e7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,6 +141,7 @@ option(FLB_SIGNV4 "Enable AWS Signv4 support" Yes) option(FLB_AWS "Enable AWS support" Yes) option(FLB_STATIC_CONF "Build binary using static configuration") option(FLB_STREAM_PROCESSOR "Enable Stream Processor" Yes) +option(FLB_SIMD "Enable SIMD support" No) option(FLB_CORO_STACK_SIZE "Set coroutine stack size") option(FLB_AVRO_ENCODER "Build with Avro encoding support" No) option(FLB_AWS_ERROR_REPORTER "Build with aws error reporting support" No) @@ -181,6 +182,10 @@ option(FLB_EVENT_LOOP_KQUEUE "Enable kqueue(2) event loop backend" No) option(FLB_EVENT_LOOP_SELECT "Enable select(2) event loop backend" No) option(FLB_EVENT_LOOP_LIBEVENT "Enable libevent event loop backend" No) +# SIMD support +if(FLB_SIMD) + FLB_DEFINITION(FLB_HAVE_SIMD) +endif() if(DEFINED FLB_NIGHTLY_BUILD AND NOT "${FLB_NIGHTLY_BUILD}" STREQUAL "") FLB_DEFINITION_VAL(FLB_NIGHTLY_BUILD ${FLB_NIGHTLY_BUILD}) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 734471ea61e..a1a8aef3fb2 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -711,7 +711,8 @@ The dependencies must be present: * Microsoft Visual Studio C/C++ toolchain. The CI automation uses MSVC 2019 at time of writing. MSVC Community Edition works fine. * [CMake](https://cmake.org/) 3.x on the `PATH` -* A build of [OpenSSL](https://www.openssl.org/) as static libraries, pointed to by the `-DOPENSSL_ROOT_DIR` CMake variable. The CI automation uses [Chocolatey](https://chocolatey.org/) to `choco install -y openssl`. +* A build of [OpenSSL](https://www.openssl.org/) as static libraries, pointed to by the `-DOPENSSL_ROOT_DIR` CMake variable. +* The CI automation uses vcpkg to install dependencies [](https://github.com/fluent/fluent-bit/blob/master/.github/workflows/call-build-windows.yaml#L148) * `flex.exe` and `bison.exe` must be present on the `PATH`. The CI automation uses https://github.com/lexxmark/winflexbison. Assuming that `cmake` is on the `PATH`, Visual Studio is installed, @@ -731,6 +732,134 @@ The build output will be `bin\Debug\fluent-bit.exe`. If in doubt, check the CI and build automation files referenced above for specifics. +### Building on a Windows Server 2022 + +The following steps have been tested on a Windows Server 2022 Datacenter edition on top of GCP. + +1. **Download and Install Visual Studio 2022** (Community Edition) + - **Download**: Go to [Visual Studio Download Page](https://visualstudio.microsoft.com/downloads/). + - **Install**: + - Select **Community Edition** and check the following components during installation: + - **Desktop development with C++** + - **Linux development with C++** + +2. **Install Flex and Bison** + 1. Create a new file called `setup-flex-bison.ps1` and paste the following script: + + ```powershell + # Define variables for Flex and Bison + $flexBisonUrl = "https://sourceforge.net/projects/winflexbison/files/win_flex_bison3-latest.zip/download" + $downloadPath = "$env:TEMP\win_flex_bison.zip" + $extractPath = "C:\win_flex_bison" + $flexExe = "flex.exe" + $bisonExe = "bison.exe" + + # Step 2: Download and Setup Flex and Bison + Write-Output "Downloading win_flex_bison..." + Invoke-WebRequest -Uri $flexBisonUrl -OutFile $downloadPath + + # Create the extract directory if it does not exist + If (!(Test-Path -Path $extractPath)) { + New-Item -ItemType Directory -Path $extractPath + } + + # Extract the zip file + Write-Output "Extracting win_flex_bison..." + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($downloadPath, $extractPath) + + # Rename the executables + Write-Output "Renaming executables..." + Rename-Item "$extractPath\win_flex.exe" "$extractPath\$flexExe" -Force + Rename-Item "$extractPath\win_bison.exe" "$extractPath\$bisonExe" -Force + + # Add Flex and Bison path to system environment variables + Write-Output "Adding Flex and Bison path to environment variables..." + $envPath = [System.Environment]::GetEnvironmentVariable("Path", "Machine") + If ($envPath -notlike "*$extractPath*") { + [System.Environment]::SetEnvironmentVariable("Path", "$envPath;$extractPath", "Machine") + Write-Output "Path updated. Please restart your command prompt to apply changes." + } else { + Write-Output "Path already contains the Flex and Bison directory." + } + + # Cleanup + Remove-Item $downloadPath + + Write-Output "Flex and Bison setup complete." + ``` + + 2. Run the Script: Open PowerShell as administrator. + + ```powershell + .\setup-flex-bison.ps1 + ``` + + 3. Restart the command prompt: After the script completes, restart your command prompt or Visual Studio for the changes to take effect. + +3. **Create `vcpkg.json` file for the dependencies** + + In the root of your project, create a `vcpkg.json` file with the following content: + + ```json + { + "name": "fluent-bit", + "version": "3.2.0", + "dependencies": [ + { + "name": "openssl", + "default-features": false + }, + { + "name": "libyaml", + "default-features": false + } + ], + "builtin-baseline": "9f5925e81bbcd9c8c34cc7a8bd25e3c557b582b2" + } + ``` + +4. **Install dependencies using `vcpkg`** + + ```bash + vcpkg install --triplet x64-windows-static + ``` + + You should see output like: + + ```bash + libyaml:x64-windows-static 0.2.5#5 A C library for parsing and emitting YAML. + openssl:x64-windows-static 3.3.2#1 OpenSSL is an open source project that provides SSL and TLS. + ``` + +5. **Link `vcpkg` with Visual Studio** + + ```bash + vcpkg integrate install + ``` + +6. **Generate the Visual Studio solution of Fluent Bit using CMake** + + ```bash + cd build + cmake -G "Visual Studio 17 2022" -DFLB_TESTS_INTERNAL=Off -DFLB_TESTS_RUNTIME=Off -DCMAKE_TOOLCHAIN_FILE="C:/Program Files/Microsoft Visual Studio/2022/Community/VC/vcpkg/scripts/buildsystems/vcpkg.cmake" -DOPENSSL_ROOT_DIR=C:/path/to/your/vcpkg_installed/x64-windows-static -DFLB_LIBYAML_DIR=C:/path/to/your/vcpkg_installed/x64-windows-static .. + ``` + + **Notes**: + - Replace `C:/path/to/your/vcpkg_installed/x64-windows-static` with the actual path where `vcpkg` installed OpenSSL and LibYAML. + - When installing with `vcpkg`, you can also specify a different install root using `--x-install-root`. + - This will generate a Visual Studio solution file, which you can open and compile. + +7. **Run the binary build** + + ```bash + cmake --build . --parallel 4 --clean-first + ``` + + **Notes**: + - You can choose to omit the `--parallel` option. + - The `--clean-first` option will clear cache and start a fresh clean build. + ### Valgrind [Valgrind](https://valgrind.org/) is a tool that will help you detect and diagnose memory issues in your code. It will check for memory leaks and invalid memory accesses. diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h index 37d50594bb5..b5ae20fa327 100644 --- a/include/fluent-bit/config_format/flb_cf.h +++ b/include/fluent-bit/config_format/flb_cf.h @@ -55,7 +55,10 @@ enum cf_file_format { enum section_type { FLB_CF_SERVICE = 0, /* [SERVICE] */ FLB_CF_PARSER, /* [PARSER] */ - FLB_CF_MULTILINE_PARSER, /* [MULTILINE_PARSER] */ + FLB_CF_MULTILINE_PARSER, /* multiline_parser */ + FLB_CF_STREAM_PROCESSOR, /* stream_processor */ + FLB_CF_PLUGINS, /* plugins */ + FLB_CF_UPSTREAM_SERVERS, /* upstream_servers */ FLB_CF_CUSTOM, /* [CUSTOM] */ FLB_CF_INPUT, /* [INPUT] */ FLB_CF_FILTER, /* [FILTER] */ @@ -97,7 +100,16 @@ struct flb_cf { struct mk_list parsers; struct mk_list multiline_parsers; - /* custom plugins */ + /* stream processor: every entry is added as a task */ + struct mk_list stream_processors; + + /* external plugins (.so) */ + struct mk_list plugins; + + /* upstream servers */ + struct mk_list upstream_servers; + + /* 'custom' type plugins */ struct mk_list customs; /* pipeline */ diff --git a/include/fluent-bit/flb_parser.h b/include/fluent-bit/flb_parser.h index 3c92ca489ea..afc8159b100 100644 --- a/include/fluent-bit/flb_parser.h +++ b/include/fluent-bit/flb_parser.h @@ -75,7 +75,7 @@ enum { FLB_PARSER_TYPE_HEX, }; -static inline time_t flb_parser_tm2time(const struct flb_tm *src, +static inline time_t flb_parser_tm2time(const struct flb_tm *src, int use_system_timezone) { struct tm tmp; @@ -107,6 +107,11 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, struct flb_config *config); int flb_parser_conf_file_stat(const char *file, struct flb_config *config); int flb_parser_conf_file(const char *file, struct flb_config *config); +int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, + struct flb_config *config); +int flb_parser_load_multiline_parser_definitions(const char *cfg, struct flb_cf *cf, + struct flb_config *config); + void flb_parser_destroy(struct flb_parser *parser); struct flb_parser *flb_parser_get(const char *name, struct flb_config *config); int flb_parser_do(struct flb_parser *parser, const char *buf, size_t length, diff --git a/include/fluent-bit/flb_plugin.h b/include/fluent-bit/flb_plugin.h index 44369c4a363..af78b1cde91 100644 --- a/include/fluent-bit/flb_plugin.h +++ b/include/fluent-bit/flb_plugin.h @@ -47,8 +47,12 @@ struct flb_plugins { struct flb_plugins *flb_plugin_create(); int flb_plugin_load(char *path, struct flb_plugins *ctx, struct flb_config *config); + int flb_plugin_load_router(char *path, struct flb_config *config); + int flb_plugin_load_config_file(const char *file, struct flb_config *config); +int flb_plugin_load_config_format(struct flb_cf *cf, struct flb_config *config); + void flb_plugin_destroy(struct flb_plugins *ctx); #endif diff --git a/include/fluent-bit/flb_simd.h b/include/fluent-bit/flb_simd.h new file mode 100644 index 00000000000..9ae53b115fe --- /dev/null +++ b/include/fluent-bit/flb_simd.h @@ -0,0 +1,262 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_SIMD_H +#define FLB_SIMD_H + +#include +#include + +#include +#define UINT64CONST(x) (x##ULL) + +/* Only enable SIMD support if it has not been explicity disabled */ +#ifdef FLB_HAVE_SIMD + +#if (defined(__x86_64__) || defined(_M_AMD64)) +/* + * SSE2 instructions are part of the spec for the 64-bit x86 ISA. We assume + * that compilers targeting this architecture understand SSE2 intrinsics. + * + * We use emmintrin.h rather than the comprehensive header immintrin.h in + * order to exclude extensions beyond SSE2. This is because MSVC, at least, + * will allow the use of intrinsics that haven't been enabled at compile + * time. + */ +#include +#define FLB_SIMD_SSE2 + +typedef __m128i flb_vector8; +typedef __m128i flb_vector32; + +#elif defined(__aarch64__) && defined(__ARM_NEON) +/* + * We use the Neon instructions if the compiler provides access to them (as + * indicated by __ARM_NEON) and we are on aarch64. While Neon support is + * technically optional for aarch64, it appears that all available 64-bit + * hardware does have it. Neon exists in some 32-bit hardware too, but we + * could not realistically use it there without a run-time check, which seems + * not worth the trouble for now. + */ +#include +#define FLB_SIMD_NEON +typedef uint8x16_t flb_vector8; +typedef uint32x4_t flb_vector32; + +#else +/* + * If no SIMD instructions are available, we can in some cases emulate vector + * operations using bitwise operations on unsigned integers. Note that many + * of the functions in this file presently do not have non-SIMD + * implementations. In particular, none of the functions involving Vector32 + * are implemented without SIMD since it's likely not worthwhile to represent + * two 32-bit integers using a uint64. + */ +#define FLB_SIMD_NONE +typedef uint64_t flb_vector8; +#endif + +#else +#define FLB_SIMD_NONE + +/* Original code aims to handle this as a uint64_t to search */ +typedef uint8_t flb_vector8; +#endif /* FLB_SIMD_DISABLED */ + +/* element-wise comparisons to a scalar */ +static inline bool flb_vector8_has(const flb_vector8 v, const uint8_t c); +static inline bool flb_vector8_has_zero(const flb_vector8 v); +static inline bool flb_vector8_has_le(const flb_vector8 v, const uint8_t c); +static inline bool flb_vector8_is_highbit_set(const flb_vector8 v); + +/* + * Load a chunk of memory into the given vector. + */ +static inline void flb_vector8_load(flb_vector8 *v, const uint8_t *s) +{ +#if defined(FLB_SIMD_SSE2) + *v = _mm_loadu_si128((const __m128i *) s); +#elif defined(FLB_SIMD_NEON) + *v = vld1q_u8(s); +#else + memset(v, 0, sizeof(flb_vector8)); +#endif +} + +/* + * Convenience function equivalent to vector8_has(v, 0) + */ +static inline bool flb_vector8_has_zero(const flb_vector8 v) +{ +#if defined(FLB_SIMD_NONE) + /* + * We cannot call vector8_has() here, because that would lead to a + * circular definition. + */ + return flb_vector8_has_le(v, 0); +#else + return flb_vector8_has(v, 0); +#endif +} + + +/* + * Return the result of subtracting the respective elements of the input + * vectors using saturation (i.e., if the operation would yield a value less + * than zero, zero is returned instead). For more information on saturation + * arithmetic, see https://en.wikipedia.org/wiki/Saturation_arithmetic + */ +#ifndef FLB_SIMD_NONE +static inline flb_vector8 flb_vector8_ssub(const flb_vector8 v1, const flb_vector8 v2) +{ +#ifdef FLB_SIMD_SSE2 + return _mm_subs_epu8(v1, v2); +#elif defined(FLB_SIMD_NEON) + return vqsubq_u8(v1, v2); +#endif +} +#endif /* ! FLB_SIMD_NONE */ + +/* + * Return a vector with all bits set in each lane where the corresponding + * lanes in the inputs are equal. + */ +#ifndef FLB_SIMD_NONE +static inline flb_vector8 flb_vector8_eq(const flb_vector8 v1, const flb_vector8 v2) +{ +#ifdef FLB_SIMD_SSE2 + return _mm_cmpeq_epi8(v1, v2); +#elif defined(FLB_SIMD_NEON) + return vceqq_u8(v1, v2); +#endif +} +#endif /* ! FLB_SIMD_NONE */ + +#ifndef FLB_SIMD_NONE +static inline flb_vector32 flb_vector32_eq(const flb_vector32 v1, const flb_vector32 v2) +{ +#ifdef FLB_SIMD_SSE2 + return _mm_cmpeq_epi32(v1, v2); +#elif defined(FLB_SIMD_NEON) + return vceqq_u32(v1, v2); +#endif +} +#endif /* ! FLB_SIMD_NONE */ + +/* + * Create a vector with all elements set to the same value. + */ +static inline flb_vector8 flb_vector8_broadcast(const uint8_t c) +{ +#if defined(FLB_SIMD_SSE2) + return _mm_set1_epi8(c); +#elif defined(FLB_SIMD_NEON) + return vdupq_n_u8(c); +#else + return ~UINT64CONST(0) / 0xFF * c; +#endif +} + +/* + * Return true if the high bit of any element is set + */ +static inline bool flb_vector8_is_highbit_set(const flb_vector8 v) +{ +#ifdef FLB_SIMD_SSE2 + return _mm_movemask_epi8(v) != 0; +#elif defined(FLB_SIMD_NEON) + return vmaxvq_u8(v) > 0x7F; +#else + return v & flb_vector8_broadcast(0x80); +#endif +} + +/* + * Return true if any elements in the vector are equal to the given scalar. + */ +static inline bool flb_vector8_has(const flb_vector8 v, const uint8_t c) +{ + bool result = false; + +#if defined(FLB_SIMD_NONE) + return flb_vector8_has_zero(v ^ flb_vector8_broadcast(c)); +#else + result = flb_vector8_is_highbit_set(flb_vector8_eq(v, flb_vector8_broadcast(c))); +#endif + + return result; +} + +static inline bool flb_vector8_has_le(const flb_vector8 v, const uint8_t c) +{ + bool result = false; + +#if defined(FLB_SIMD_NONE) + /* + * To find bytes <= c, we can use bitwise operations to find bytes < c+1, + * but it only works if c+1 <= 128 and if the highest bit in v is not set. + * + * Adapted from + * + * https://graphics.stanford.edu/~seander/bithacks.html#HasLessInWord + */ + if ((int64_t) v >= 0 && c < 0x80) { + result = (v - flb_vector8_broadcast(c + 1)) & ~v & flb_vector8_broadcast(0x80); + } + else { + size_t i; + for (i = 0; i < sizeof(flb_vector8); i++) { + if (((const uint8_t *) &v)[i] <= c) { + result = true; + break; + } + } + } + + return result; +#else + /* + * Use saturating subtraction to find bytes <= c, which will present as + * NUL bytes. This approach is a workaround for the lack of unsigned + * comparison instructions on some architectures. + */ + result = flb_vector8_has_zero(flb_vector8_ssub(v, flb_vector8_broadcast(c))); +#endif + + return result; +} + +static inline char *flb_simd_info() +{ + #ifdef FLB_HAVE_SIMD + #if defined(FLB_SIMD_SSE2) + return "SSE2"; + #elif defined(FLB_SIMD_NEON) + return "NEON"; + #elif defined(FLB_SIMD_NONE) + return "none"; + #else + return "unknown"; + #endif + #else + return "disabled"; + #endif +} + +#endif /* FLB_HAVE_SIMD */ diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index 8081a9a42fe..c3fb8918e62 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -561,6 +561,11 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, (uint64_t) fs_entry_metadata.st_ino); } else { + /* Result codes : + * 0 - Success + * -1 - Generic failure + * 1 - The file was alrady present in our records + */ flb_plg_debug(ctx->ins, "blob scan skip: %s", glob_context.gl_pathv[index]); diff --git a/plugins/in_blob/blob_file.c b/plugins/in_blob/blob_file.c index bf5aaf5caca..0a51bd9de81 100644 --- a/plugins/in_blob/blob_file.c +++ b/plugins/in_blob/blob_file.c @@ -40,7 +40,7 @@ int blob_file_append(struct blob_ctx *ctx, char *path, struct stat *st) bfile = cfl_list_entry(head, struct blob_file, _head); if (strcmp(bfile->path, path) == 0) { /* file already exists */ - return -1; + return 1; } } @@ -48,7 +48,7 @@ int blob_file_append(struct blob_ctx *ctx, char *path, struct stat *st) if (ctx->database_file) { /* the file was already registered, just skipt it */ if (blob_db_file_exists(ctx, path, &id_found) == FLB_TRUE) { - return 0; + return 1; } } #endif diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index de4ee41beaf..98a1ff8807f 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -649,6 +649,12 @@ static int process_blob_chunk(struct flb_azure_blob *ctx, struct flb_event_chunk struct flb_log_event_decoder log_decoder; struct flb_log_event log_event; + if (ctx->db == NULL) { + flb_plg_error(ctx->ins, "Cannot process blob because this operation requires a database."); + + return -1; + } + ret = flb_log_event_decoder_init(&log_decoder, (char *) event_chunk->data, event_chunk->size); @@ -668,7 +674,8 @@ static int process_blob_chunk(struct flb_azure_blob *ctx, struct flb_event_chunk continue; } - ret = azb_db_file_insert(ctx, source, ctx->endpoint, file_path, file_size); + ret = azb_db_file_insert(ctx, source, ctx->real_endpoint, file_path, file_size); + if (ret == -1) { flb_plg_error(ctx->ins, "cannot insert blob file into database: %s (size=%lu)", file_path, file_size); @@ -723,6 +730,10 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context flb_sched_timer_cb_coro_return(); } + if (ctx->db == NULL) { + flb_sched_timer_cb_coro_return(); + } + info->active_upload = FLB_TRUE; /* @@ -893,10 +904,13 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context /* just continue, the row info was retrieved */ } - if (strcmp(file_destination, ctx->endpoint) != 0) { + + if (strcmp(file_destination, ctx->real_endpoint) != 0) { flb_plg_info(ctx->ins, - "endpoint change detected, restarting file : %s", - file_path); + "endpoint change detected, restarting file : %s\n%s\n%s", + file_path, + file_destination, + ctx->real_endpoint); info->active_upload = FLB_FALSE; diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index a66af75933a..d9c6ee68837 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -567,6 +567,11 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in return NULL; } + if (ctx->account_name == NULL) { + flb_plg_error(ctx->ins, "'account_name' has not been set"); + return NULL; + } + if (ctx->configuration_endpoint_url != NULL) { ret = flb_azure_blob_apply_remote_configuration(ctx); diff --git a/plugins/out_azure_blob/azure_blob_db.c b/plugins/out_azure_blob/azure_blob_db.c index 9ba5e0948d3..6951877ae01 100644 --- a/plugins/out_azure_blob/azure_blob_db.c +++ b/plugins/out_azure_blob/azure_blob_db.c @@ -251,6 +251,10 @@ struct flb_sqldb *azb_db_open(struct flb_azure_blob *ctx, char *db_path) int azb_db_close(struct flb_azure_blob *ctx) { + if (ctx->db == NULL) { + return 0; + } + /* finalize prepared statements */ sqlite3_finalize(ctx->stmt_insert_file); sqlite3_finalize(ctx->stmt_delete_file); @@ -272,6 +276,7 @@ int azb_db_close(struct flb_azure_blob *ctx) sqlite3_finalize(ctx->stmt_get_oldest_file_with_parts); pthread_mutex_destroy(&ctx->db_lock); + return flb_sqldb_close(ctx->db); } @@ -751,7 +756,7 @@ int azb_db_file_part_get_next(struct flb_azure_blob *ctx, *part_delivery_attempts = sqlite3_column_int64(ctx->stmt_get_next_file_part, 5); tmp = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 6); *file_delivery_attempts = sqlite3_column_int64(ctx->stmt_get_next_file_part, 7); - tmp_destination = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 8); + tmp_destination = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 9); } else if (ret == SQLITE_DONE) { /* no records */ diff --git a/plugins/processor_content_modifier/CMakeLists.txt b/plugins/processor_content_modifier/CMakeLists.txt index d3ea9d24a10..681d2cd58df 100644 --- a/plugins/processor_content_modifier/CMakeLists.txt +++ b/plugins/processor_content_modifier/CMakeLists.txt @@ -1,7 +1,10 @@ set(src cm_config.c cm_logs.c + cm_metrics.c cm_traces.c + cm_opentelemetry.c + cm_utils.c cm.c ) diff --git a/plugins/processor_content_modifier/cm.c b/plugins/processor_content_modifier/cm.c index 4a2cf444793..4c56d659ff6 100644 --- a/plugins/processor_content_modifier/cm.c +++ b/plugins/processor_content_modifier/cm.c @@ -98,6 +98,24 @@ static int cb_process_traces(struct flb_processor_instance *ins, } +static int cb_process_metrics(struct flb_processor_instance *ins, + struct cmt *in_cmt, + struct cmt **out_cmt, + const char *tag, + int tag_len) +{ + int ret; + struct content_modifier_ctx *ctx; + + if (!ins->context) { + return FLB_PROCESSOR_FAILURE; + } + ctx = ins->context; + + ret = cm_metrics_process(ins, ctx, in_cmt, out_cmt, tag, tag_len); + return ret; +} + static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "context", NULL, @@ -144,7 +162,7 @@ struct flb_processor_plugin processor_content_modifier_plugin = { .description = "Modify the content of Logs, Metrics and Traces", .cb_init = cb_init, .cb_process_logs = cb_process_logs, - .cb_process_metrics = NULL, + .cb_process_metrics = cb_process_metrics, .cb_process_traces = cb_process_traces, .cb_exit = cb_exit, .config_map = config_map, diff --git a/plugins/processor_content_modifier/cm.h b/plugins/processor_content_modifier/cm.h index c6cae0b6b34..181872b0754 100644 --- a/plugins/processor_content_modifier/cm.h +++ b/plugins/processor_content_modifier/cm.h @@ -116,5 +116,10 @@ int cm_traces_process(struct flb_processor_instance *ins, struct ctrace *traces_context, const char *tag, int tag_len); +int cm_metrics_process(struct flb_processor_instance *ins, + struct content_modifier_ctx *ctx, + struct cmt *in_cmt, + struct cmt **out_cmt, + const char *tag, int tag_len); #endif diff --git a/plugins/processor_content_modifier/cm_config.c b/plugins/processor_content_modifier/cm_config.c index 487977cb113..70ba9915715 100644 --- a/plugins/processor_content_modifier/cm_config.c +++ b/plugins/processor_content_modifier/cm_config.c @@ -218,6 +218,53 @@ static int set_context(struct content_modifier_ctx *ctx) strcasecmp(ctx->context_str, "attributes") == 0) { context = CM_CONTEXT_METRIC_LABELS; } + + /* + * OpenTelemetry contexts + * ---------------------- + */ + else if (strcasecmp(ctx->context_str, "otel_resource_attributes") == 0) { + context = CM_CONTEXT_OTEL_RESOURCE_ATTR; + } + else if (strcasecmp(ctx->context_str, "otel_scope_attributes") == 0) { + context = CM_CONTEXT_OTEL_SCOPE_ATTR; + } + else if (strcasecmp(ctx->context_str, "otel_scope_name") == 0) { + /* + * scope name is restricted to specific actions, make sure the user + * cannot messed it up + * + * action allowed ? + * ----------------------------- + * CM_ACTION_INSERT Yes + * CM_ACTION_UPSERT Yes + * CM_ACTION_DELETE Yes + * CM_ACTION_RENAME No + * CM_ACTION_HASH Yes + * CM_ACTION_EXTRACT No + * CM_ACTION_CONVERT No + */ + + if (ctx->action_type == CM_ACTION_RENAME || + ctx->action_type == CM_ACTION_EXTRACT || + ctx->action_type == CM_ACTION_CONVERT) { + flb_plg_error(ctx->ins, "action '%s' is not allowed for context '%s'", + ctx->action_str, ctx->context_str); + return -1; + } + + /* check that 'name' is the key set */ + if (!ctx->key) { + ctx->key = flb_sds_create("name"); + } + else if (strcasecmp(ctx->key, "name") != 0) { + flb_plg_error(ctx->ins, "context '%s' requires the name of the key to be 'name', no '%s'", + ctx->context_str, ctx->key); + return -1; + } + + context = CM_CONTEXT_OTEL_SCOPE_NAME; + } else { flb_plg_error(ctx->ins, "unknown metrics context '%s'", ctx->context_str); return -1; diff --git a/plugins/processor_content_modifier/cm_logs.c b/plugins/processor_content_modifier/cm_logs.c index 278540039b5..49e6e2fe08d 100644 --- a/plugins/processor_content_modifier/cm_logs.c +++ b/plugins/processor_content_modifier/cm_logs.c @@ -22,345 +22,15 @@ #include #include #include -#include #include #include #include #include "cm.h" #include "cm_utils.h" +#include "cm_opentelemetry.h" #include -#include - -static int hex_encode(unsigned char *input_buffer, - size_t input_length, - cfl_sds_t *output_buffer) -{ - const char hex[] = "0123456789abcdef"; - cfl_sds_t result; - size_t index; - - if (cfl_sds_alloc(*output_buffer) <= (input_length * 2)) { - result = cfl_sds_increase(*output_buffer, - (input_length * 2) - - cfl_sds_alloc(*output_buffer)); - - if (result == NULL) { - return FLB_FALSE; - } - - *output_buffer = result; - } - - for (index = 0; index < input_length; index++) { - (*output_buffer)[index * 2 + 0] = hex[(input_buffer[index] >> 4) & 0xF]; - (*output_buffer)[index * 2 + 1] = hex[(input_buffer[index] >> 0) & 0xF]; - } - - cfl_sds_set_len(*output_buffer, input_length * 2); - - (*output_buffer)[index * 2] = '\0'; - - return FLB_TRUE; -} - -static int hash_transformer(void *context, struct cfl_variant *value) -{ - unsigned char digest_buffer[32]; - struct cfl_variant *converted_value; - cfl_sds_t encoded_hash; - int result; - - if (value == NULL) { - return FLB_FALSE; - } - - result = cfl_variant_convert(value, - &converted_value, - CFL_VARIANT_STRING); - - if (result != FLB_TRUE) { - return FLB_FALSE; - } - - if (cfl_variant_size_get(converted_value) == 0) { - cfl_variant_destroy(converted_value); - return FLB_TRUE; - } - - result = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) converted_value->data.as_string, - cfl_sds_len(converted_value->data.as_string), - digest_buffer, - sizeof(digest_buffer)); - - if (result != FLB_CRYPTO_SUCCESS) { - cfl_variant_destroy(converted_value); - - return FLB_FALSE; - } - - result = hex_encode(digest_buffer, - sizeof(digest_buffer), - &converted_value->data.as_string); - - if (result != FLB_TRUE) { - cfl_variant_destroy(converted_value); - - return FLB_FALSE; - } - - encoded_hash = cfl_sds_create(converted_value->data.as_string); - cfl_variant_destroy(converted_value); - if (encoded_hash == NULL) { - return FLB_FALSE; - } - - /* NOTE: this part does a manual modification of the variant content */ - if (value->type == CFL_VARIANT_STRING || - value->type == CFL_VARIANT_BYTES) { - if (value->referenced == CFL_FALSE) { - cfl_sds_destroy(value->data.as_string); - } - } - else if (value->type == CFL_VARIANT_ARRAY) { - cfl_array_destroy(value->data.as_array); - } - else if (value->type == CFL_VARIANT_KVLIST) { - cfl_kvlist_destroy(value->data.as_kvlist); - } - - value->type = CFL_VARIANT_STRING; - value->data.as_string = encoded_hash; - value->referenced = CFL_FALSE; - - cfl_variant_size_set(value, cfl_sds_len(encoded_hash)); - - return FLB_TRUE; -} - -cfl_sds_t cfl_variant_convert_to_json(struct cfl_variant *value) -{ - cfl_sds_t json_result; - mpack_writer_t writer; - char *data; - size_t size; - - data = NULL; - size = 0; - - mpack_writer_init_growable(&writer, &data, &size); - - pack_cfl_variant(&writer, value); - - mpack_writer_destroy(&writer); - - json_result = flb_msgpack_raw_to_json_sds(data, size); - - return json_result; -} - -int cfl_variant_convert(struct cfl_variant *input_value, - struct cfl_variant **output_value, - int output_type) -{ - int ret; - int errno_backup; - int64_t as_int; - double as_double; - char buf[64]; - char *str = NULL; - char *converstion_canary = NULL; - struct cfl_variant *tmp = NULL; - - errno_backup = errno; - - /* input: string, bytes or reference */ - if (input_value->type == CFL_VARIANT_STRING || input_value->type == CFL_VARIANT_BYTES || - input_value->type == CFL_VARIANT_REFERENCE) { - - if (output_type == CFL_VARIANT_STRING || - output_type == CFL_VARIANT_BYTES) { - - tmp = cfl_variant_create_from_string_s(input_value->data.as_string, - cfl_variant_size_get(input_value), - CFL_FALSE); - if (!tmp) { - return CFL_FALSE; - } - } - else if (output_type == CFL_VARIANT_BOOL) { - as_int = CFL_FALSE; - - if (cfl_variant_size_get(input_value) == 4 && - strncasecmp(input_value->data.as_string, "true", 4) == 0) { - as_int = CFL_TRUE; - } - else if (cfl_variant_size_get(input_value) == 5 && - strncasecmp(input_value->data.as_string, "false", 5) == 0) { - as_int = CFL_FALSE; - } - - tmp = cfl_variant_create_from_bool(as_int); - } - else if (output_type == CFL_VARIANT_INT) { - errno = 0; - - if (input_value->referenced) { - tmp = cfl_variant_create_from_string_s(input_value->data.as_string, - cfl_variant_size_get(input_value), - CFL_FALSE); - if (!tmp) { - return CFL_FALSE; - } - str = tmp->data.as_string; - } - else { - str = input_value->data.as_string; - } - - as_int = strtoimax(str, &converstion_canary, 10); - if (errno == ERANGE || errno == EINVAL) { - errno = errno_backup; - if (tmp) { - cfl_variant_destroy(tmp); - } - return CFL_FALSE; - } - - if (tmp) { - cfl_variant_destroy(tmp); - } - - tmp = cfl_variant_create_from_int64(as_int); - } - else if (output_type == CFL_VARIANT_DOUBLE) { - errno = 0; - converstion_canary = NULL; - - if (input_value->referenced) { - tmp = cfl_variant_create_from_string_s(input_value->data.as_string, - cfl_variant_size_get(input_value), - CFL_FALSE); - if (!tmp) { - return CFL_FALSE; - } - str = tmp->data.as_string; - } - else { - str = input_value->data.as_string; - } - - as_double = strtod(str, &converstion_canary); - if (errno == ERANGE) { - errno = errno_backup; - if (tmp) { - cfl_variant_destroy(tmp); - } - return CFL_FALSE; - } - - if (tmp) { - cfl_variant_destroy(tmp); - } - - if (as_double == 0 && converstion_canary == input_value->data.as_string) { - errno = errno_backup; - return CFL_FALSE; - } - - tmp = cfl_variant_create_from_double(as_double); - } - else { - return CFL_FALSE; - } - } - /* input: int */ - else if (input_value->type == CFL_VARIANT_INT) { - if (output_type == CFL_VARIANT_STRING || output_type == CFL_VARIANT_BYTES) { - ret = snprintf(buf, sizeof(buf), "%" PRIi64, input_value->data.as_int64); - if (ret < 0 || ret >= sizeof(buf)) { - return CFL_FALSE; - } - tmp = cfl_variant_create_from_string_s(buf, ret, CFL_FALSE); - } - else if (output_type == CFL_VARIANT_BOOL) { - as_int = CFL_FALSE; - if (input_value->data.as_int64 != 0) { - as_int = CFL_TRUE; - } - - tmp = cfl_variant_create_from_bool(as_int); - } - else if (output_type == CFL_VARIANT_INT) { - /* same type, do nothing */ - } - else if (output_type == CFL_VARIANT_DOUBLE) { - as_double = (double) input_value->data.as_int64; - tmp = cfl_variant_create_from_double(as_double); - } - else { - return CFL_FALSE; - } - } - else if (input_value->type == CFL_VARIANT_DOUBLE) { - if (output_type == CFL_VARIANT_STRING || - output_type == CFL_VARIANT_BYTES) { - - ret = snprintf(buf, sizeof(buf), "%.17g", input_value->data.as_double); - if (ret < 0 || ret >= sizeof(buf)) { - return CFL_FALSE; - } - tmp = cfl_variant_create_from_string_s(buf, ret, CFL_FALSE); - } - else if (output_type == CFL_VARIANT_BOOL) { - as_int = CFL_FALSE; - - if (input_value->data.as_double != 0) { - as_int = CFL_TRUE; - } - - tmp = cfl_variant_create_from_bool(as_int); - } - else if (output_type == CFL_VARIANT_INT) { - as_int = (int64_t) round(input_value->data.as_double); - tmp = cfl_variant_create_from_int64(as_int); - } - else if (output_type == CFL_VARIANT_DOUBLE) { - as_double = input_value->data.as_int64; - tmp = cfl_variant_create_from_double(as_double); - } - else { - return CFL_FALSE; - } - } - else if (input_value->type == CFL_VARIANT_NULL) { - if (output_type == CFL_VARIANT_STRING || - output_type == CFL_VARIANT_BYTES) { - - tmp = cfl_variant_create_from_string_s("null", 4, CFL_FALSE); - } - else if (output_type == CFL_VARIANT_BOOL) { - tmp = cfl_variant_create_from_bool(CFL_FALSE); - } - else if (output_type == CFL_VARIANT_INT) { - tmp = cfl_variant_create_from_int64(0); - } - else if (output_type == CFL_VARIANT_DOUBLE) { - tmp = cfl_variant_create_from_double(0); - } - else { - return CFL_FALSE; - } - } - else { - return CFL_FALSE; - } - - *output_value = tmp; - return FLB_TRUE; -} static struct cfl_kvpair *cfl_object_kvpair_get(struct cfl_object *obj, cfl_sds_t key) { @@ -501,7 +171,7 @@ static int run_action_hash(struct content_modifier_ctx *ctx, return 0; } - ret = hash_transformer(NULL, kvpair->val); + ret = cm_utils_hash_transformer(NULL, kvpair->val); if (ret == FLB_FALSE) { return -1; } @@ -581,7 +251,7 @@ static int run_action_convert(struct content_modifier_ctx *ctx, /* convert the value */ v = kvpair->val; - ret = cfl_variant_convert(v, &converted, converted_type); + ret = cm_utils_variant_convert(v, &converted, converted_type); if (ret != FLB_TRUE) { return -1; } @@ -599,134 +269,29 @@ static int run_action_convert(struct content_modifier_ctx *ctx, return 0; } -static struct cfl_variant *otel_get_or_create_attributes(struct cfl_kvlist *kvlist) -{ - int ret; - struct cfl_list *head; - struct cfl_list *tmp; - struct cfl_kvpair *kvpair; - struct cfl_variant *val; - struct cfl_kvlist *kvlist_tmp; - - /* iterate resource to find the attributes field */ - cfl_list_foreach_safe(head, tmp, &kvlist->list) { - kvpair = cfl_list_entry(head, struct cfl_kvpair, _head); - if (cfl_sds_len(kvpair->key) != 10) { - continue; - } - - if (strncmp(kvpair->key, "attributes", 10) == 0) { - val = kvpair->val; - if (val->type != CFL_VARIANT_KVLIST) { - return NULL; - } - - return val; - } - } - - /* create an empty kvlist as the value of attributes */ - kvlist_tmp = cfl_kvlist_create(); - if (!kvlist_tmp) { - return NULL; - } - - /* create the attributes kvpair */ - ret = cfl_kvlist_insert_kvlist_s(kvlist, "attributes", 10, kvlist_tmp); - if (ret != 0) { - cfl_kvlist_destroy(kvlist_tmp); - return NULL; - } - - /* get the last kvpair from the list */ - kvpair = cfl_list_entry_last(&kvlist->list, struct cfl_kvpair, _head); - if (!kvpair) { - return NULL; - } - - return kvpair->val; -} - static struct cfl_variant *otel_get_attributes(int context, struct flb_mp_chunk_record *record) { - int key_len; - const char *key_buf; - struct cfl_list *head; struct cfl_object *obj = NULL; - struct cfl_variant *val; struct cfl_kvlist *kvlist; - struct cfl_kvpair *kvpair; - struct cfl_variant *var_attr; - - if (context == CM_CONTEXT_OTEL_RESOURCE_ATTR) { - key_buf = "resource"; - key_len = 8; - } - else if (context == CM_CONTEXT_OTEL_SCOPE_ATTR) { - key_buf = "scope"; - key_len = 5; - } - else { - return NULL; - } obj = record->cobj_record; kvlist = obj->variant->data.as_kvlist; - cfl_list_foreach(head, &kvlist->list) { - kvpair = cfl_list_entry(head, struct cfl_kvpair, _head); - - if (cfl_sds_len(kvpair->key) != key_len) { - continue; - } - - if (strncmp(kvpair->key, key_buf, key_len) == 0) { - val = kvpair->val; - if (val->type != CFL_VARIANT_KVLIST) { - return NULL; - } - var_attr = otel_get_or_create_attributes(val->data.as_kvlist); - if (!var_attr) { - return NULL; - } - - return var_attr; - } - } - - return NULL; + return cm_otel_get_attributes(CM_TELEMETRY_LOGS, context, kvlist); } static struct cfl_variant *otel_get_scope(struct flb_mp_chunk_record *record) { - struct cfl_list *head; struct cfl_object *obj; - struct cfl_variant *val; struct cfl_kvlist *kvlist; - struct cfl_kvpair *kvpair; obj = record->cobj_record; kvlist = obj->variant->data.as_kvlist; - cfl_list_foreach(head, &kvlist->list) { - kvpair = cfl_list_entry(head, struct cfl_kvpair, _head); - if (cfl_sds_len(kvpair->key) != 5) { - continue; - } - - if (strncmp(kvpair->key, "scope", 5) == 0) { - val = kvpair->val; - if (val->type != CFL_VARIANT_KVLIST) { - return NULL; - } - - return val; - } - } - - return NULL; + return cm_otel_get_scope_metadata(CM_TELEMETRY_LOGS, kvlist); } + int cm_logs_process(struct flb_processor_instance *ins, struct content_modifier_ctx *ctx, struct flb_mp_chunk_cobj *chunk_cobj, diff --git a/plugins/processor_content_modifier/cm_metrics.c b/plugins/processor_content_modifier/cm_metrics.c new file mode 100644 index 00000000000..d4e97aec533 --- /dev/null +++ b/plugins/processor_content_modifier/cm_metrics.c @@ -0,0 +1,346 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "cm.h" +#include "cm_utils.h" +#include "cm_opentelemetry.h" + + +static struct cfl_kvpair *kvlist_get_kvpair(struct cfl_kvlist *kvlist, cfl_sds_t key) +{ + struct cfl_list *head; + struct cfl_kvpair *kvpair; + + cfl_list_foreach(head, &kvlist->list) { + kvpair = cfl_list_entry(head, struct cfl_kvpair, _head); + + if (cfl_sds_len(key) != cfl_sds_len(kvpair->key)) { + continue; + } + + if (strncmp(key, kvpair->key, cfl_sds_len(key)) == 0) { + return kvpair; + } + } + + return NULL; +} + +static int run_action_insert(struct content_modifier_ctx *ctx, + struct cfl_kvlist *kvlist, + const char *tag, int tag_len, + cfl_sds_t key, cfl_sds_t value) +{ + int ret; + struct cfl_kvpair *kvpair; + + /* check that the key don't exists */ + kvpair = kvlist_get_kvpair(kvlist, key); + if (kvpair) { + /* Insert requires the key don't exists, we fail silently */ + return 0; + } + + /* insert the new value */ + ret = cfl_kvlist_insert_string_s(kvlist, key, cfl_sds_len(key), value, cfl_sds_len(value), + CFL_FALSE); + if (ret != 0) { + flb_plg_debug(ctx->ins, "[action: insert] failed to insert key: %s", key); + return -1; + } + return 0; +} + +static int run_action_upsert(struct content_modifier_ctx *ctx, + struct cfl_kvlist *kvlist, + const char *tag, int tag_len, + cfl_sds_t key, cfl_sds_t value) +{ + int ret; + struct cfl_kvpair *kvpair; + + /* if the kv pair already exists, remove it from the list */ + kvpair = kvlist_get_kvpair(kvlist, key); + if (kvpair) { + cfl_kvpair_destroy(kvpair); + } + + /* insert the key with the updated value */ + ret = cfl_kvlist_insert_string_s(kvlist, key, cfl_sds_len(key), value, cfl_sds_len(value), + CFL_FALSE); + if (ret != 0) { + return -1; + } + + return 0; +} + +static int run_action_delete(struct content_modifier_ctx *ctx, + struct cfl_kvlist *kvlist, + const char *tag, int tag_len, + cfl_sds_t key) +{ + struct cfl_kvpair *kvpair; + + /* if the kv pair already exists, remove it from the list */ + kvpair = kvlist_get_kvpair(kvlist, key); + if (kvpair) { + cfl_kvpair_destroy(kvpair); + return 0; + } + + flb_plg_debug(ctx->ins, "[action: delete] key '%s' not found", key); + + /* if the kvpair was not found, it's ok, we return zero */ + return 0; +} + +static int run_action_rename(struct content_modifier_ctx *ctx, + struct cfl_kvlist *kvlist, + const char *tag, int tag_len, + cfl_sds_t key, cfl_sds_t value) +{ + cfl_sds_t tmp; + struct cfl_kvpair *kvpair; + + /* if the kv pair already exists, remove it from the list */ + kvpair = kvlist_get_kvpair(kvlist, key); + if (!kvpair) { + flb_plg_debug(ctx->ins, "[action: rename] key '%s' not found", key); + return 0; + } + + tmp = kvpair->key; + + kvpair->key = cfl_sds_create_len(value, cfl_sds_len(value)); + if (!kvpair->key) { + /* restore previous value */ + kvpair->key = tmp; + return -1; + } + + /* destroy previous value */ + cfl_sds_destroy(tmp); + return 0; +} + +static int run_action_hash(struct content_modifier_ctx *ctx, + struct cfl_kvlist *kvlist, + const char *tag, int tag_len, + cfl_sds_t key) +{ + int ret; + struct cfl_kvpair *kvpair; + + /* if the kv pair already exists, remove it from the list */ + kvpair = kvlist_get_kvpair(kvlist, key); + if (!kvpair) { + /* the key was not found, so it's ok */ + return 0; + } + + ret = cm_utils_hash_transformer(NULL, kvpair->val); + if (ret == FLB_FALSE) { + return -1; + } + + return 0; +} + +static void cb_extract_regex(const char *name, const char *value, size_t value_length, void *context) +{ + + struct cfl_kvlist *kvlist = (struct cfl_kvlist *) context; + + if (cfl_kvlist_contains(kvlist, (char *) name)) { + cfl_kvlist_remove(kvlist, (char *) name); + } + + cfl_kvlist_insert_string_s(kvlist, (char *) name, strlen(name), (char *) value, value_length, + CFL_FALSE); +} + +static int run_action_extract(struct content_modifier_ctx *ctx, + struct cfl_kvlist *kvlist, + const char *tag, int tag_len, + cfl_sds_t key, struct flb_regex *regex) +{ + int ret; + int match_count; + struct flb_regex_search match_list; + struct cfl_kvpair *kvpair; + struct cfl_variant *v; + + /* if the kv pair already exists, remove it from the list */ + kvpair = kvlist_get_kvpair(kvlist, key); + if (!kvpair) { + return -1; + } + + v = kvpair->val; + if (v->type != CFL_VARIANT_STRING) { + return -1; + } + + match_count = flb_regex_do(regex, + v->data.as_string, + cfl_variant_size_get(v), &match_list); + if (match_count <= 0) { + return -1; + } + + ret = flb_regex_parse(regex, &match_list, cb_extract_regex, kvlist); + if (ret == -1) { + return -1; + } + + return 0; +} + +static int run_action_convert(struct content_modifier_ctx *ctx, + struct cfl_kvlist *kvlist, + const char *tag, int tag_len, + cfl_sds_t key, int converted_type) +{ + int ret; + struct cfl_variant *v; + struct cfl_kvpair *kvpair; + struct cfl_variant *converted; + + /* if the kv pair already exists, remove it from the list */ + kvpair = kvlist_get_kvpair(kvlist, key); + if (!kvpair) { + return -1; + } + + /* convert the value */ + v = kvpair->val; + ret = cm_utils_variant_convert(v, &converted, converted_type); + if (ret != FLB_TRUE) { + return -1; + } + + /* remove the old kvpair */ + cfl_kvpair_destroy(kvpair); + + ret = cfl_kvlist_insert_s(kvlist, key, cfl_sds_len(key), converted); + if (ret != 0) { + cfl_variant_destroy(converted); + return -1; + } + + return 0; +} + +int cm_metrics_process(struct flb_processor_instance *ins, + struct content_modifier_ctx *ctx, + struct cmt *in_cmt, + struct cmt **out_cmt, + const char *tag, int tag_len) +{ + int ret = -1; + struct cfl_variant *var = NULL; + + printf("\n\n==== BEFORE =====\n"); + cfl_kvlist_print(stdout, in_cmt->internal_metadata); + printf("\n"); + printf("-----external----\n"); + cfl_kvlist_print(stdout, in_cmt->external_metadata); + fflush(stdout); + + + if (ctx->context_type == CM_CONTEXT_OTEL_RESOURCE_ATTR) { + /* Internal metadata must be valid */ + var = cfl_kvlist_fetch(in_cmt->internal_metadata, "producer"); + if (!var) { + return FLB_PROCESSOR_FAILURE; + } + + if (var->type != CFL_VARIANT_STRING) { + return FLB_PROCESSOR_FAILURE; + } + + /* validate that the value is 'opentelemetry' */ + if (strcmp(var->data.as_string, "opentelemetry") != 0) { + return FLB_PROCESSOR_FAILURE; + } + + /* Now check the external metadata */ + if (!in_cmt->external_metadata) { + return FLB_PROCESSOR_FAILURE; + } + + var = NULL; + + var = cm_otel_get_attributes(CM_TELEMETRY_METRICS, ctx->context_type, in_cmt->external_metadata); + if (!var) { + return FLB_PROCESSOR_FAILURE; + } + } + else if (ctx->context_type == CM_CONTEXT_OTEL_SCOPE_ATTR) { + var = cm_otel_get_attributes(CM_TELEMETRY_METRICS, ctx->context_type, in_cmt->external_metadata); + } + else if ((ctx->context_type == CM_CONTEXT_OTEL_SCOPE_NAME || ctx->context_type == CM_CONTEXT_OTEL_SCOPE_VERSION)) { + var = cm_otel_get_scope_metadata(CM_TELEMETRY_METRICS, in_cmt->external_metadata); + } + + if (!var) { + return FLB_PROCESSOR_FAILURE; + } + + if (ctx->action_type == CM_ACTION_INSERT) { + ret = run_action_insert(ctx, var->data.as_kvlist, tag, tag_len, ctx->key, ctx->value); + } + else if (ctx->action_type == CM_ACTION_UPSERT) { + ret = run_action_upsert(ctx, var->data.as_kvlist, tag, tag_len, ctx->key, ctx->value); + } + else if (ctx->action_type == CM_ACTION_DELETE) { + ret = run_action_delete(ctx, var->data.as_kvlist, tag, tag_len, ctx->key); + } + else if (ctx->action_type == CM_ACTION_RENAME) { + ret = run_action_rename(ctx, var->data.as_kvlist, tag, tag_len, ctx->key, ctx->value); + } + else if (ctx->action_type == CM_ACTION_HASH) { + ret = run_action_hash(ctx, var->data.as_kvlist, tag, tag_len, ctx->key); + } + else if (ctx->action_type == CM_ACTION_EXTRACT) { + ret = run_action_extract(ctx, var->data.as_kvlist, tag, tag_len, ctx->key, ctx->regex); + } + else if (ctx->action_type == CM_ACTION_CONVERT) { + ret = run_action_convert(ctx, var->data.as_kvlist, tag, tag_len, ctx->key, ctx->converted_type); + } + + if (ret != 0) { + return FLB_PROCESSOR_FAILURE; + } + + printf("\n\n==== AFTER =====\n"); + cfl_kvlist_print(stdout, in_cmt->internal_metadata); + printf("\n"); + printf("-----external----\n"); + cfl_kvlist_print(stdout, in_cmt->external_metadata); + fflush(stdout); + + return FLB_PROCESSOR_SUCCESS; +} diff --git a/plugins/processor_content_modifier/cm_opentelemetry.c b/plugins/processor_content_modifier/cm_opentelemetry.c new file mode 100644 index 00000000000..0a37a89a2b5 --- /dev/null +++ b/plugins/processor_content_modifier/cm_opentelemetry.c @@ -0,0 +1,272 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cm.h" +#include "cm_utils.h" + +#include + +struct cfl_variant *cm_otel_get_or_create_attributes(struct cfl_kvlist *kvlist) +{ + int ret; + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_kvpair *kvpair; + struct cfl_variant *val; + struct cfl_kvlist *kvlist_tmp; + + /* iterate resource to find the attributes field */ + cfl_list_foreach_safe(head, tmp, &kvlist->list) { + kvpair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (cfl_sds_len(kvpair->key) != 10) { + continue; + } + + if (strncmp(kvpair->key, "attributes", 10) == 0) { + val = kvpair->val; + if (val->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return val; + } + } + + /* create an empty kvlist as the value of attributes */ + kvlist_tmp = cfl_kvlist_create(); + if (!kvlist_tmp) { + return NULL; + } + + /* create the attributes kvpair */ + ret = cfl_kvlist_insert_kvlist_s(kvlist, "attributes", 10, kvlist_tmp); + if (ret != 0) { + cfl_kvlist_destroy(kvlist_tmp); + return NULL; + } + + /* get the last kvpair from the list */ + kvpair = cfl_list_entry_last(&kvlist->list, struct cfl_kvpair, _head); + if (!kvpair) { + return NULL; + } + + return kvpair->val; +} + +static struct cfl_variant *otel_get_or_create_attributes(struct cfl_kvlist *kvlist) +{ + int ret; + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_kvpair *kvpair; + struct cfl_variant *val = NULL; + struct cfl_kvlist *kvlist_tmp; + + /* iterate resource to find the attributes field */ + cfl_list_foreach_safe(head, tmp, &kvlist->list) { + kvpair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (cfl_sds_len(kvpair->key) != 10) { + continue; + } + + if (strncmp(kvpair->key, "attributes", 10) == 0) { + val = kvpair->val; + if (val->type != CFL_VARIANT_KVLIST) { + return NULL; + } + return val; + } + } + + /* create an empty kvlist as the value of attributes */ + kvlist_tmp = cfl_kvlist_create(); + if (!kvlist_tmp) { + return NULL; + } + + /* create the attributes kvpair */ + ret = cfl_kvlist_insert_kvlist_s(kvlist, "attributes", 10, kvlist_tmp); + if (ret != 0) { + cfl_kvlist_destroy(kvlist_tmp); + return NULL; + } + + /* get the last kvpair from the list */ + kvpair = cfl_list_entry_last(&kvlist->list, struct cfl_kvpair, _head); + if (!kvpair) { + return NULL; + } + + return kvpair->val; +} + +/* + * get attributes for resources and scope, context must be one of: + * + * - CM_CONTEXT_OTEL_RESOURCE_ATTR + * - CM_CONTEXT_OTEL_SCOPE_ATTR + */ +struct cfl_variant *cm_otel_get_attributes(int telemetry_type, int context, struct cfl_kvlist *kvlist) +{ + int key_len; + const char *key_buf; + struct cfl_variant *var; + struct cfl_variant *var_attr = NULL; + + if (context == CM_CONTEXT_OTEL_RESOURCE_ATTR) { + key_buf = "resource"; + key_len = 8; + } + else if (context == CM_CONTEXT_OTEL_SCOPE_ATTR) { + key_buf = "scope"; + key_len = 5; + } + else { + return NULL; + } + + var = cfl_kvlist_fetch_s(kvlist, (char *) key_buf, key_len); + if (!var) { + return NULL; + } + + if (var->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + var_attr = otel_get_or_create_attributes(var->data.as_kvlist); + if (!var_attr) { + return NULL; + } + + return var_attr; +} + +static struct cfl_variant *otel_get_or_create_scope_metadata(int telemetry_type, struct cfl_kvlist *kvlist) +{ + int ret; + struct cfl_variant *var; + struct cfl_kvpair *kvpair; + struct cfl_kvlist *kvlist_tmp; + + /* kvlist is the value of 'scope', lookup for scope->metadata */ + + var = cfl_kvlist_fetch(kvlist, "metadata"); + if (var) { + if (var->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return var; + } + + /* metadata don't exists, create it */ + kvlist_tmp = cfl_kvlist_create(); + if (!kvlist_tmp) { + return NULL; + } + + ret = cfl_kvlist_insert_kvlist_s(kvlist, "metadata", 8, kvlist_tmp); + if (ret != 0) { + cfl_kvlist_destroy(kvlist_tmp); + return NULL; + } + + kvpair = cfl_list_entry_last(&kvlist->list, struct cfl_kvpair, _head); + if (!kvpair) { + return NULL; + } + + return kvpair->val; +} + +/* + * Retrieve the kvlist that contains the scope metadata such as name and version, + * based on the telemetry type, the kvlist is expected to be in the following format: + * + * - Logs: scope -> {name, version} + * - Metrics: scope -> metadata -> {name, version} + * + * If the paths are not found, those are "created". + */ +struct cfl_variant *cm_otel_get_scope_metadata(int telemetry_type, struct cfl_kvlist *kvlist) +{ + int ret; + struct cfl_variant *var = NULL; + struct cfl_kvpair *kvpair = NULL; + struct cfl_kvlist *kvlist_tmp; + + if (!kvlist) { + return NULL; + } + + /* retrieve the scope if exists */ + var = cfl_kvlist_fetch(kvlist, "scope"); + if (var) { + if (var->type != CFL_VARIANT_KVLIST) { + /* if exists and is not valid just fail */ + return NULL; + } + } + else { + /* create "scope" inside kvlist */ + kvlist_tmp = cfl_kvlist_create(); + if (!kvlist_tmp) { + return NULL; + } + + ret = cfl_kvlist_insert_kvlist_s(kvlist, "scope", 5, kvlist_tmp); + if (ret != 0) { + cfl_kvlist_destroy(kvlist_tmp); + return NULL; + } + + kvpair = cfl_list_entry_last(&kvlist->list, struct cfl_kvpair, _head); + if (!kvpair) { + return NULL; + } + + var = kvpair->val; + } + + /* + * 'var' points to the value of 'scope', for logs telemetry data, just return + * the current variant, for metrics lookup for 'metadata' kvpair (or create it) + */ + + if (telemetry_type == CM_TELEMETRY_LOGS) { + return var; + } + else if (telemetry_type == CM_TELEMETRY_METRICS) { + var = otel_get_or_create_scope_metadata(telemetry_type, var->data.as_kvlist); + } + + return var; +} diff --git a/plugins/processor_content_modifier/cm_opentelemetry.h b/plugins/processor_content_modifier/cm_opentelemetry.h new file mode 100644 index 00000000000..ddd8823f61b --- /dev/null +++ b/plugins/processor_content_modifier/cm_opentelemetry.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PROCESSOR_CONTENT_MODIFIER_OTEL_H +#define FLB_PROCESSOR_CONTENT_MODIFIER_OTEL_H + +#include + +struct cfl_variant *cm_otel_get_attributes(int telemetry_type, int context, struct cfl_kvlist *kvlist); +struct cfl_variant *cm_otel_get_scope_metadata(int telemetry_type, struct cfl_kvlist *kvlist); + +#endif \ No newline at end of file diff --git a/plugins/processor_content_modifier/cm_traces.c b/plugins/processor_content_modifier/cm_traces.c index 24783ba37fc..54d4469bcb6 100644 --- a/plugins/processor_content_modifier/cm_traces.c +++ b/plugins/processor_content_modifier/cm_traces.c @@ -23,8 +23,6 @@ #include #include -//#include "variant_utils.h" - #include "cm.h" #include "cm_utils.h" @@ -117,9 +115,9 @@ static int span_convert_attribute(struct ctrace_span *span, return FLB_FALSE; } - ret = cfl_variant_convert(attribute, - &converted_attribute, - new_type); + ret = cm_utils_variant_convert(attribute, + &converted_attribute, + new_type); if (ret != FLB_TRUE) { return FLB_FALSE; @@ -262,107 +260,6 @@ static int context_contains_attribute(struct ctrace *traces_context, return FLB_FALSE; } -static int hex_encode(unsigned char *input_buffer, - size_t input_length, - cfl_sds_t *output_buffer) -{ - const char hex[] = "0123456789abcdef"; - cfl_sds_t result; - size_t index; - - if (cfl_sds_alloc(*output_buffer) <= (input_length * 2)) { - result = cfl_sds_increase(*output_buffer, - (input_length * 2) - - cfl_sds_alloc(*output_buffer)); - - if (result == NULL) { - return FLB_FALSE; - } - - *output_buffer = result; - } - - for (index = 0; index < input_length; index++) { - (*output_buffer)[index * 2 + 0] = hex[(input_buffer[index] >> 4) & 0xF]; - (*output_buffer)[index * 2 + 1] = hex[(input_buffer[index] >> 0) & 0xF]; - } - - cfl_sds_set_len(*output_buffer, input_length * 2); - - (*output_buffer)[index * 2] = '\0'; - - return FLB_TRUE; -} - -static int hash_transformer(void *context, struct cfl_variant *value) -{ - unsigned char digest_buffer[32]; - struct cfl_variant *converted_value; - cfl_sds_t encoded_hash; - int result; - - if (value == NULL) { - return FLB_FALSE; - } - - result = cfl_variant_convert(value, - &converted_value, - CFL_VARIANT_STRING); - - if (result != FLB_TRUE) { - return FLB_FALSE; - } - - if (cfl_sds_len(converted_value->data.as_string) == 0) { - cfl_variant_destroy(converted_value); - return FLB_TRUE; - } - - result = flb_hash_simple(FLB_HASH_SHA256, - (unsigned char *) converted_value->data.as_string, - cfl_sds_len(converted_value->data.as_string), - digest_buffer, - sizeof(digest_buffer)); - - if (result != FLB_CRYPTO_SUCCESS) { - cfl_variant_destroy(converted_value); - return FLB_FALSE; - } - - result = hex_encode(digest_buffer, - sizeof(digest_buffer), - &converted_value->data.as_string); - - if (result != FLB_TRUE) { - cfl_variant_destroy(converted_value); - return FLB_FALSE; - } - - encoded_hash = cfl_sds_create(converted_value->data.as_string); - if (encoded_hash == NULL) { - cfl_variant_destroy(converted_value); - return FLB_FALSE; - } - cfl_variant_destroy(converted_value); - - - if (value->type == CFL_VARIANT_STRING || - value->type == CFL_VARIANT_BYTES) { - cfl_sds_destroy(value->data.as_string); - } - else if (value->type == CFL_VARIANT_ARRAY) { - cfl_array_destroy(value->data.as_array); - } - else if (value->type == CFL_VARIANT_KVLIST) { - cfl_kvlist_destroy(value->data.as_kvlist); - } - - value->type = CFL_VARIANT_STRING; - value->data.as_string = encoded_hash; - - return FLB_TRUE; -} - static int traces_context_hash_attribute(struct ctrace *traces_context, char *name) { @@ -374,7 +271,7 @@ static int traces_context_hash_attribute(struct ctrace *traces_context, struct ctrace_span, _head_global); if (span_contains_attribute(span, name) == FLB_TRUE) { - if (span_transform_attribute(span, name, hash_transformer) != FLB_TRUE) { + if (span_transform_attribute(span, name, cm_utils_hash_transformer) != FLB_TRUE) { return FLB_FALSE; } } diff --git a/plugins/processor_content_modifier/cm_utils.c b/plugins/processor_content_modifier/cm_utils.c new file mode 100644 index 00000000000..7d762f6c4bc --- /dev/null +++ b/plugins/processor_content_modifier/cm_utils.c @@ -0,0 +1,357 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "cm.h" +#include "cm_utils.h" + +#include +static int hex_encode(unsigned char *input_buffer, size_t input_length, cfl_sds_t *output_buffer) +{ + const char hex[] = "0123456789abcdef"; + cfl_sds_t result; + size_t index; + + if (cfl_sds_alloc(*output_buffer) <= (input_length * 2)) { + result = cfl_sds_increase(*output_buffer, + (input_length * 2) - + cfl_sds_alloc(*output_buffer)); + + if (result == NULL) { + return FLB_FALSE; + } + + *output_buffer = result; + } + + for (index = 0; index < input_length; index++) { + (*output_buffer)[index * 2 + 0] = hex[(input_buffer[index] >> 4) & 0xF]; + (*output_buffer)[index * 2 + 1] = hex[(input_buffer[index] >> 0) & 0xF]; + } + + cfl_sds_set_len(*output_buffer, input_length * 2); + + (*output_buffer)[index * 2] = '\0'; + + return FLB_TRUE; +} + + +int cm_utils_hash_transformer(void *context, struct cfl_variant *value) +{ + unsigned char digest_buffer[32]; + struct cfl_variant *converted_value; + cfl_sds_t encoded_hash; + int result; + + if (value == NULL) { + return FLB_FALSE; + } + + result = cm_utils_variant_convert(value, + &converted_value, + CFL_VARIANT_STRING); + + if (result != FLB_TRUE) { + return FLB_FALSE; + } + + if (cfl_variant_size_get(converted_value) == 0) { + cfl_variant_destroy(converted_value); + return FLB_TRUE; + } + + result = flb_hash_simple(FLB_HASH_SHA256, + (unsigned char *) converted_value->data.as_string, + cfl_sds_len(converted_value->data.as_string), + digest_buffer, + sizeof(digest_buffer)); + + if (result != FLB_CRYPTO_SUCCESS) { + cfl_variant_destroy(converted_value); + + return FLB_FALSE; + } + + result = hex_encode(digest_buffer, + sizeof(digest_buffer), + &converted_value->data.as_string); + + if (result != FLB_TRUE) { + cfl_variant_destroy(converted_value); + + return FLB_FALSE; + } + + encoded_hash = cfl_sds_create(converted_value->data.as_string); + cfl_variant_destroy(converted_value); + if (encoded_hash == NULL) { + return FLB_FALSE; + } + + /* NOTE: this part does a manual modification of the variant content */ + if (value->type == CFL_VARIANT_STRING || + value->type == CFL_VARIANT_BYTES) { + if (value->referenced == CFL_FALSE) { + cfl_sds_destroy(value->data.as_string); + } + } + else if (value->type == CFL_VARIANT_ARRAY) { + cfl_array_destroy(value->data.as_array); + } + else if (value->type == CFL_VARIANT_KVLIST) { + cfl_kvlist_destroy(value->data.as_kvlist); + } + + value->type = CFL_VARIANT_STRING; + value->data.as_string = encoded_hash; + value->referenced = CFL_FALSE; + + cfl_variant_size_set(value, cfl_sds_len(encoded_hash)); + + return FLB_TRUE; +} + +cfl_sds_t cm_utils_variant_convert_to_json(struct cfl_variant *value) +{ + cfl_sds_t json_result; + mpack_writer_t writer; + char *data; + size_t size; + + data = NULL; + size = 0; + + mpack_writer_init_growable(&writer, &data, &size); + + pack_cfl_variant(&writer, value); + + mpack_writer_destroy(&writer); + + json_result = flb_msgpack_raw_to_json_sds(data, size); + + return json_result; +} + +int cm_utils_variant_convert(struct cfl_variant *input_value, + struct cfl_variant **output_value, + int output_type) +{ + int ret; + int errno_backup; + int64_t as_int; + double as_double; + char buf[64]; + char *str = NULL; + char *converstion_canary = NULL; + struct cfl_variant *tmp = NULL; + + errno_backup = errno; + + /* input: string, bytes or reference */ + if (input_value->type == CFL_VARIANT_STRING || input_value->type == CFL_VARIANT_BYTES || + input_value->type == CFL_VARIANT_REFERENCE) { + + if (output_type == CFL_VARIANT_STRING || + output_type == CFL_VARIANT_BYTES) { + + tmp = cfl_variant_create_from_string_s(input_value->data.as_string, + cfl_variant_size_get(input_value), + CFL_FALSE); + if (!tmp) { + return CFL_FALSE; + } + } + else if (output_type == CFL_VARIANT_BOOL) { + as_int = CFL_FALSE; + + if (cfl_variant_size_get(input_value) == 4 && + strncasecmp(input_value->data.as_string, "true", 4) == 0) { + as_int = CFL_TRUE; + } + else if (cfl_variant_size_get(input_value) == 5 && + strncasecmp(input_value->data.as_string, "false", 5) == 0) { + as_int = CFL_FALSE; + } + + tmp = cfl_variant_create_from_bool(as_int); + } + else if (output_type == CFL_VARIANT_INT) { + errno = 0; + + if (input_value->referenced) { + tmp = cfl_variant_create_from_string_s(input_value->data.as_string, + cfl_variant_size_get(input_value), + CFL_FALSE); + if (!tmp) { + return CFL_FALSE; + } + str = tmp->data.as_string; + } + else { + str = input_value->data.as_string; + } + + as_int = strtoimax(str, &converstion_canary, 10); + if (errno == ERANGE || errno == EINVAL) { + errno = errno_backup; + if (tmp) { + cfl_variant_destroy(tmp); + } + return CFL_FALSE; + } + + if (tmp) { + cfl_variant_destroy(tmp); + } + + tmp = cfl_variant_create_from_int64(as_int); + } + else if (output_type == CFL_VARIANT_DOUBLE) { + errno = 0; + converstion_canary = NULL; + + if (input_value->referenced) { + tmp = cfl_variant_create_from_string_s(input_value->data.as_string, + cfl_variant_size_get(input_value), + CFL_FALSE); + if (!tmp) { + return CFL_FALSE; + } + str = tmp->data.as_string; + } + else { + str = input_value->data.as_string; + } + + as_double = strtod(str, &converstion_canary); + if (errno == ERANGE) { + errno = errno_backup; + if (tmp) { + cfl_variant_destroy(tmp); + } + return CFL_FALSE; + } + + if (tmp) { + cfl_variant_destroy(tmp); + } + + if (as_double == 0 && converstion_canary == input_value->data.as_string) { + errno = errno_backup; + return CFL_FALSE; + } + + tmp = cfl_variant_create_from_double(as_double); + } + else { + return CFL_FALSE; + } + } + /* input: int */ + else if (input_value->type == CFL_VARIANT_INT) { + if (output_type == CFL_VARIANT_STRING || output_type == CFL_VARIANT_BYTES) { + ret = snprintf(buf, sizeof(buf), "%" PRIi64, input_value->data.as_int64); + if (ret < 0 || ret >= sizeof(buf)) { + return CFL_FALSE; + } + tmp = cfl_variant_create_from_string_s(buf, ret, CFL_FALSE); + } + else if (output_type == CFL_VARIANT_BOOL) { + as_int = CFL_FALSE; + if (input_value->data.as_int64 != 0) { + as_int = CFL_TRUE; + } + + tmp = cfl_variant_create_from_bool(as_int); + } + else if (output_type == CFL_VARIANT_INT) { + /* same type, do nothing */ + } + else if (output_type == CFL_VARIANT_DOUBLE) { + as_double = (double) input_value->data.as_int64; + tmp = cfl_variant_create_from_double(as_double); + } + else { + return CFL_FALSE; + } + } + else if (input_value->type == CFL_VARIANT_DOUBLE) { + if (output_type == CFL_VARIANT_STRING || + output_type == CFL_VARIANT_BYTES) { + + ret = snprintf(buf, sizeof(buf), "%.17g", input_value->data.as_double); + if (ret < 0 || ret >= sizeof(buf)) { + return CFL_FALSE; + } + tmp = cfl_variant_create_from_string_s(buf, ret, CFL_FALSE); + } + else if (output_type == CFL_VARIANT_BOOL) { + as_int = CFL_FALSE; + + if (input_value->data.as_double != 0) { + as_int = CFL_TRUE; + } + + tmp = cfl_variant_create_from_bool(as_int); + } + else if (output_type == CFL_VARIANT_INT) { + as_int = (int64_t) round(input_value->data.as_double); + tmp = cfl_variant_create_from_int64(as_int); + } + else if (output_type == CFL_VARIANT_DOUBLE) { + as_double = input_value->data.as_int64; + tmp = cfl_variant_create_from_double(as_double); + } + else { + return CFL_FALSE; + } + } + else if (input_value->type == CFL_VARIANT_NULL) { + if (output_type == CFL_VARIANT_STRING || + output_type == CFL_VARIANT_BYTES) { + + tmp = cfl_variant_create_from_string_s("null", 4, CFL_FALSE); + } + else if (output_type == CFL_VARIANT_BOOL) { + tmp = cfl_variant_create_from_bool(CFL_FALSE); + } + else if (output_type == CFL_VARIANT_INT) { + tmp = cfl_variant_create_from_int64(0); + } + else if (output_type == CFL_VARIANT_DOUBLE) { + tmp = cfl_variant_create_from_double(0); + } + else { + return CFL_FALSE; + } + } + else { + return CFL_FALSE; + } + + *output_value = tmp; + return FLB_TRUE; +} diff --git a/plugins/processor_content_modifier/cm_utils.h b/plugins/processor_content_modifier/cm_utils.h index 7116172603c..bec26741b07 100644 --- a/plugins/processor_content_modifier/cm_utils.h +++ b/plugins/processor_content_modifier/cm_utils.h @@ -629,5 +629,10 @@ int cfl_variant_convert(struct cfl_variant *input_value, int output_type); +int cm_utils_hash_transformer(void *context, struct cfl_variant *value); +cfl_sds_t cm_utils_variant_convert_to_json(struct cfl_variant *value); +int cm_utils_variant_convert(struct cfl_variant *input_value, + struct cfl_variant **output_value, + int output_type); #endif diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c index e09034393f3..609f9b7aafd 100644 --- a/src/config_format/flb_cf_yaml.c +++ b/src/config_format/flb_cf_yaml.c @@ -57,6 +57,12 @@ enum section { SECTION_FILTER, SECTION_OUTPUT, SECTION_PROCESSOR, + SECTION_PARSER, + SECTION_MULTILINE_PARSER, + SECTION_MULTILINE_PARSER_RULE, + SECTION_STREAM_PROCESSOR, + SECTION_PLUGINS, + SECTION_UPSTREAM_SERVERS, SECTION_OTHER, }; @@ -70,6 +76,12 @@ static char *section_names[] = { "filter", "output", "processor", + "parser", + "multiline_parser", + "multiline_parser_rule", + "stream_processor", + "plugins", + "upstream_servers", "other" }; @@ -130,6 +142,34 @@ enum state { STATE_INPUT_PROCESSORS, STATE_INPUT_PROCESSOR, + /* Parser */ + STATE_PARSER, /* parser section */ + STATE_PARSER_ENTRY, /* a parser definition */ + STATE_PARSER_KEY, /* reading a key inside a parser */ + STATE_PARSER_VALUE, /* reading a value inside a parser */ + + /* Multiline Parser */ + STATE_MULTILINE_PARSER, /* multiline parser section */ + STATE_MULTILINE_PARSER_ENTRY, /* a multiline parser definition */ + STATE_MULTILINE_PARSER_VALUE, /* reading a value inside a multiline parser */ + STATE_MULTILINE_PARSER_RULE, /* reading a multiline parser rule */ + + /* Stream Processor */ + STATE_STREAM_PROCESSOR, + STATE_STREAM_PROCESSOR_ENTRY, + STATE_STREAM_PROCESSOR_KEY, + + /* Plugins */ + STATE_PLUGINS, + + /* Upstream Servers */ + STATE_UPSTREAM_SERVERS, + STATE_UPSTREAM_SERVER, + STATE_UPSTREAM_SERVER_VALUE, + STATE_UPSTREAM_NODE_GROUP, + STATE_UPSTREAM_NODE, + STATE_UPSTREAM_NODE_VALUE, + /* environment variables */ STATE_ENV, @@ -152,17 +192,22 @@ struct parser_state { /* active section */ struct flb_cf_section *cf_section; + /* active group */ struct flb_cf_group *cf_group; /* key value */ flb_sds_t key; + /* section key/value list */ struct cfl_kvlist *keyvals; + /* pointer to current values in a list. */ struct cfl_array *values; + /* pointer to current variant */ struct cfl_variant *variant; + /* if the current variant is reading the key of a kvlist */ cfl_sds_t variant_kvlist_key; /* are we the owner of the values? */ @@ -252,6 +297,16 @@ static char *state_str(enum state val) return "processor"; case STATE_ENV: return "env"; + case STATE_PARSER: + return "parser"; + case STATE_MULTILINE_PARSER: + return "multiline-parser"; + case STATE_STREAM_PROCESSOR: + return "stream-processor"; + case STATE_PLUGINS: + return "plugins"; + case STATE_UPSTREAM_SERVERS: + return "upstream-servers"; case STATE_STOP: return "stop"; default: @@ -266,17 +321,35 @@ static int add_section_type(struct flb_cf *conf, struct parser_state *state) } if (state->section == SECTION_INPUT) { - state->cf_section = flb_cf_section_create(conf, "INPUT", 0); + state->cf_section = flb_cf_section_create(conf, "input", 0); } else if (state->section == SECTION_FILTER) { - state->cf_section = flb_cf_section_create(conf, "FILTER", 0); + state->cf_section = flb_cf_section_create(conf, "filter", 0); } else if (state->section == SECTION_OUTPUT) { - state->cf_section = flb_cf_section_create(conf, "OUTPUT", 0); + state->cf_section = flb_cf_section_create(conf, "output", 0); } else if (state->section == SECTION_CUSTOM) { state->cf_section = flb_cf_section_create(conf, "customs", 0); } + else if (state->section == SECTION_PARSER) { + state->cf_section = flb_cf_section_create(conf, "parser", 0); + } + else if (state->section == SECTION_MULTILINE_PARSER) { + state->cf_section = flb_cf_section_create(conf, "multiline_parser", 0); + } + else if (state->section == SECTION_STREAM_PROCESSOR) { + state->cf_section = flb_cf_section_create(conf, "stream_processor", 0); + } + else if (state->section == SECTION_PLUGINS) { + state->cf_section = flb_cf_section_create(conf, "plugins", 0); + } + else if (state->section == SECTION_UPSTREAM_SERVERS) { + state->cf_section = flb_cf_section_create(conf, "upstream_servers", 0); + } + else { + state->cf_section = flb_cf_section_create(conf, "other", 0); + } if (!state->cf_section) { return -1; @@ -328,8 +401,8 @@ static char *state_get_last(struct local_ctx *ctx) return entry->str; } -static void yaml_error_event(struct local_ctx *ctx, struct parser_state *state, - yaml_event_t *event) +static void yaml_error_event_line(struct local_ctx *ctx, struct parser_state *state, + yaml_event_t *event, int line) { struct flb_slist_entry *entry; @@ -362,6 +435,10 @@ static void yaml_error_event(struct local_ctx *ctx, struct parser_state *state, event_type_str(event), event->type, state_str(state->state), state->state); } +#define yaml_error_event(ctx, state, event) \ + yaml_error_event_line(ctx, state, event, __LINE__) + + static void yaml_error_definition(struct local_ctx *ctx, struct parser_state *state, yaml_event_t *event, char *value) { @@ -560,6 +637,7 @@ static int read_glob(struct flb_cf *conf, struct local_ctx *ctx, static void print_current_state(struct local_ctx *ctx, struct parser_state *state, yaml_event_t *event) { + /* note: change this to flb_info() for debugging purposes */ flb_debug("%*s%s->%s", state->level*2, "", state_str(state->state), event_type_str(event)); } @@ -571,6 +649,8 @@ static void print_current_properties(struct parser_state *state) struct cfl_variant *var; int idx; + /* note: change flb_debug with flb_info() for debugging purposes */ + flb_debug("%*s[%s] PROPERTIES:", state->level*2, "", section_names[state->section]); cfl_list_foreach(head, &state->keyvals->list) { @@ -869,6 +949,523 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, break; /* end of 'includes' */ + /* Handle the 'parsers' section */ + case STATE_PARSER: + switch (event->type) { + case YAML_SEQUENCE_START_EVENT: + /* Start of the parsers list */ + break; + + case YAML_MAPPING_START_EVENT: + /* we handle each parser definition as a new section */ + if (add_section_type(conf, state) == -1) { + flb_error("Unable to add parsers section"); + return YAML_FAILURE; + } + + /* Start of an individual parser entry */ + state = state_push_withvals(ctx, state, STATE_PARSER_ENTRY); + if (!state) { + flb_error("Unable to allocate state for parser entry"); + return YAML_FAILURE; + } + break; + + case YAML_SEQUENCE_END_EVENT: + /* End of the parsers list */ + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + case STATE_PARSER_ENTRY: + switch (event->type) { + case YAML_SCALAR_EVENT: + /* Found a key within the parser entry */ + value = (char *) event->data.scalar.value; + state = state_push_key(ctx, STATE_PARSER_KEY, value); + if (!state) { + flb_error("Unable to allocate state for parser key"); + return YAML_FAILURE; + } + break; + + case YAML_MAPPING_END_EVENT: + /* End of an individual parser entry */ + print_current_properties(state); + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + case STATE_PARSER_KEY: + switch (event->type) { + case YAML_SCALAR_EVENT: + /* Store the value for the previous key */ + value = (char *) event->data.scalar.value; + if (flb_cf_section_property_add(conf, state->cf_section->properties, + state->key, flb_sds_len(state->key), + value, strlen(value)) < 0) { + flb_error("unable to add property"); + return YAML_FAILURE; + } + + /* Return to the parser entry state */ + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + case STATE_PARSER_VALUE: + /* unused */ + break; + + + /* + * Handle the 'multiline_parsers' section + * -------------------------------------- + */ + case STATE_MULTILINE_PARSER: + switch (event->type) { + case YAML_SEQUENCE_START_EVENT: + /* Start of the multiline parsers list */ + break; + + case YAML_MAPPING_START_EVENT: + /* we handle each multiline parser definition as a new section */ + if (add_section_type(conf, state) == -1) { + flb_error("Unable to add multiline parsers section"); + return YAML_FAILURE; + } + + /* Start of an individual multiline parser entry */ + state = state_push_withvals(ctx, state, STATE_MULTILINE_PARSER_ENTRY); + if (!state) { + flb_error("Unable to allocate state for multiline parser entry"); + return YAML_FAILURE; + } + break; + + case YAML_SEQUENCE_END_EVENT: + /* End of the multiline parsers list */ + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + case STATE_MULTILINE_PARSER_ENTRY: + switch (event->type) { + case YAML_SCALAR_EVENT: + /* Found a key within the multiline parser entry */ + value = (char *) event->data.scalar.value; + + /* start of 'rules:' sequence */ + if (strcmp(value, "rules") == 0) { + state = state_push_withvals(ctx, state, STATE_MULTILINE_PARSER_RULE); + if (state == NULL) { + flb_error("Unable to allocate state for multiline parser rules"); + return YAML_FAILURE; + } + break; + } + + /* normal key value pair for the multiline parser */ + state = state_push_key(ctx, STATE_MULTILINE_PARSER_VALUE, value); + if (!state) { + flb_error("Unable to allocate state for multiline parser key"); + return YAML_FAILURE; + } + break; + + case YAML_MAPPING_END_EVENT: + /* End of an individual multiline parser entry */ + print_current_properties(state); + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + case STATE_MULTILINE_PARSER_VALUE: + switch (event->type) { + case YAML_SCALAR_EVENT: + /* Store the value for the previous key */ + value = (char *) event->data.scalar.value; + if (flb_cf_section_property_add(conf, state->cf_section->properties, + state->key, flb_sds_len(state->key), + value, strlen(value)) < 0) { + flb_error("unable to add property"); + return YAML_FAILURE; + } + + /* Return to the multiline parser entry state */ + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + /* + * Multiline Parser "Rules" + * ------------------------ + */ + case STATE_MULTILINE_PARSER_RULE: + switch(event->type) { + case YAML_SEQUENCE_START_EVENT: + break; + case YAML_SEQUENCE_END_EVENT: + state = state_pop(ctx); + if (state == NULL) { + flb_error("no state left"); + return YAML_FAILURE; + } + break; + case YAML_MAPPING_START_EVENT: + if (state_create_group(conf, state, "rule") == YAML_FAILURE) { + flb_error("unable to create group"); + return YAML_FAILURE; + } + + state = state_push(ctx, STATE_GROUP_KEY); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + /* create group */ + state->values = flb_cf_section_property_add_list(conf, + state->cf_section->properties, + "rules", 5); + + if (state->values == NULL) { + flb_error("no values"); + return YAML_FAILURE; + } + + break; + case YAML_MAPPING_END_EVENT: + return YAML_FAILURE; + break; + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + }; + break; + + + /* + * Stream Processor + * ---------------- + */ + case STATE_STREAM_PROCESSOR: + switch (event->type) { + case YAML_SEQUENCE_START_EVENT: + break; + + case YAML_MAPPING_START_EVENT: + if (add_section_type(conf, state) == -1) { + flb_error("Unable to add parsers section"); + return YAML_FAILURE; + } + + state = state_push_withvals(ctx, state, STATE_STREAM_PROCESSOR_ENTRY); + if (!state) { + flb_error("Unable to allocate state for stream processor entry"); + return YAML_FAILURE; + } + break; + + case YAML_SEQUENCE_END_EVENT: + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + case STATE_STREAM_PROCESSOR_ENTRY: + switch (event->type) { + case YAML_SCALAR_EVENT: + value = (char *) event->data.scalar.value; + + state = state_push_key(ctx, STATE_STREAM_PROCESSOR_KEY, value); + if (!state) { + flb_error("Unable to allocate state for stream processor key"); + return YAML_FAILURE; + } + break; + + case YAML_MAPPING_END_EVENT: + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + case STATE_STREAM_PROCESSOR_KEY: + switch (event->type) { + case YAML_SCALAR_EVENT: + value = (char *) event->data.scalar.value; + + if (flb_cf_section_property_add(conf, state->cf_section->properties, + state->key, flb_sds_len(state->key), + value, strlen(value)) < 0) { + flb_error("Unable to add property"); + return YAML_FAILURE; + } + + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + /* + * Plugins: define a list of absolute paths for external plugins to load + * --------------------------------------------------------------------- + */ + case STATE_PLUGINS: + switch (event->type) { + case YAML_SEQUENCE_START_EVENT: + /* create the section */ + if (add_section_type(conf, state) == -1) { + flb_error("Unable to add parsers section"); + return YAML_FAILURE; + } + break; + + case YAML_SCALAR_EVENT: + /* Store the path as an entry in the plugins section */ + value = (char *) event->data.scalar.value; + + /* + * note that we pass an empty string as the real value since this is + * a list of items. + */ + if (flb_cf_section_property_add(conf, state->cf_section->properties, + value, strlen(value), "", 0) == NULL) { + flb_error("Unable to add plugin path"); + return YAML_FAILURE; + } + break; + + case YAML_SEQUENCE_END_EVENT: + /* Pop back to the previous state */ + state = state_pop(ctx); + if (!state) { + flb_error("No state left"); + return YAML_FAILURE; + } + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + /* + * Upstream Servers + * ---------------- + */ + case STATE_UPSTREAM_SERVERS: + switch (event->type) { + case YAML_SEQUENCE_START_EVENT: + break; + + case YAML_MAPPING_START_EVENT: + if (add_section_type(conf, state) == -1) { + flb_error("Unable to add parsers section"); + return YAML_FAILURE; + } + + state = state_push_withvals(ctx, state, STATE_UPSTREAM_SERVER); + if (!state) { + flb_error("Unable to allocate state for upstream server"); + return YAML_FAILURE; + } + break; + + case YAML_SEQUENCE_END_EVENT: + state = state_pop(ctx); + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + /* Handling individual upstream server */ + case STATE_UPSTREAM_SERVER: + switch (event->type) { + case YAML_SCALAR_EVENT: + value = (char *) event->data.scalar.value; + + if (strcmp(value, "nodes") == 0) { + state = state_push_withvals(ctx, state, STATE_UPSTREAM_NODE_GROUP); + if (!state) { + flb_error("Unable to allocate state for node group"); + return YAML_FAILURE; + } + break; + } + + /* normal key value pair for the upstream server */ + state = state_push_key(ctx, STATE_UPSTREAM_SERVER_VALUE, value); + if (!state) { + flb_error("Unable to allocate state for upstream server key"); + return YAML_FAILURE; + } + break; + + case YAML_MAPPING_END_EVENT: + state = state_pop(ctx); + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + /* Handling upstream server key-value pairs */ + case STATE_UPSTREAM_SERVER_VALUE: + if (event->type == YAML_SCALAR_EVENT) { + value = (char *) event->data.scalar.value; + if (flb_cf_section_property_add(conf, state->cf_section->properties, + state->key, flb_sds_len(state->key), + value, strlen(value)) == NULL) { + flb_error("Unable to add upstream server property"); + return YAML_FAILURE; + } + state = state_pop(ctx); + } + break; + + /* Handling node group */ + case STATE_UPSTREAM_NODE_GROUP: + switch(event->type) { + case YAML_SEQUENCE_START_EVENT: + break; + case YAML_SEQUENCE_END_EVENT: + state = state_pop(ctx); + if (state == NULL) { + flb_error("no state left"); + return YAML_FAILURE; + } + break; + case YAML_MAPPING_START_EVENT: + if (state_create_group(conf, state, "upstream_node") == YAML_FAILURE) { + flb_error("unable to create group"); + return YAML_FAILURE; + } + state = state_push(ctx, STATE_GROUP_KEY); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + break; + case YAML_MAPPING_END_EVENT: + return YAML_FAILURE; + break; + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + }; + break; + + /* Handling individual node */ + case STATE_UPSTREAM_NODE: + switch (event->type) { + case YAML_SCALAR_EVENT: + value = (char *) event->data.scalar.value; + state = state_push_key(ctx, STATE_UPSTREAM_NODE_VALUE, value); + break; + + case YAML_MAPPING_END_EVENT: + state = state_pop(ctx); + break; + + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + /* Handling node key-value pairs */ + case STATE_UPSTREAM_NODE_VALUE: + if (event->type == YAML_SCALAR_EVENT) { + value = (char *) event->data.scalar.value; + if (flb_cf_section_property_add(conf, state->cf_group->properties, + state->key, flb_sds_len(state->key), + value, strlen(value)) == NULL) { + flb_error("Unable to add node property"); + return YAML_FAILURE; + } + state = state_pop(ctx); + } + break; + /* * 'customs' * -------- @@ -906,7 +1503,7 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, case STATE_PIPELINE: switch (event->type) { case YAML_SCALAR_EVENT: - value = (char *)event->data.scalar.value; + value = (char *) event->data.scalar.value; if (strcasecmp(value, "inputs") == 0) { state = state_push_section(ctx, STATE_PLUGIN_INPUT, SECTION_INPUT); @@ -946,7 +1543,7 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, case STATE_SECTION: switch (event->type) { case YAML_SCALAR_EVENT: - value = (char *)event->data.scalar.value; + value = (char *) event->data.scalar.value; if (strcasecmp(value, "env") == 0) { state = state_push_section(ctx, STATE_ENV, SECTION_ENV); @@ -955,6 +1552,41 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, return YAML_FAILURE; } } + else if (strcasecmp(value, "parsers") == 0) { + state = state_push_section(ctx, STATE_PARSER, SECTION_PARSER); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + } + else if (strcasecmp(value, "multiline_parsers") == 0) { + state = state_push_section(ctx, STATE_MULTILINE_PARSER, SECTION_MULTILINE_PARSER); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + } + else if (strcasecmp(value, "stream_processor") == 0) { + state = state_push_section(ctx, STATE_STREAM_PROCESSOR, SECTION_STREAM_PROCESSOR); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + } + else if (strcasecmp(value, "plugins") == 0) { + state = state_push_section(ctx, STATE_PLUGINS, SECTION_PLUGINS); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + } + else if (strcasecmp(value, "upstream_servers") == 0) { + state = state_push_section(ctx, STATE_UPSTREAM_SERVERS, SECTION_UPSTREAM_SERVERS); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + } else if (strcasecmp(value, "pipeline") == 0) { state = state_push_section(ctx, STATE_PIPELINE, SECTION_PIPELINE); if (state == NULL) { @@ -985,7 +1617,7 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, else if (strcasecmp(value, "customs") == 0) { state = state_push_section(ctx, STATE_CUSTOM, SECTION_CUSTOM); - if (state == NULL) { + if (state == NULL) { flb_error("unable to allocate state"); return YAML_FAILURE; } @@ -1466,7 +2098,7 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, return YAML_FAILURE; } - if (cfl_array_append_string(state->values, (char *)event->data.scalar.value) < 0) { + if (cfl_array_append_string(state->values, (char *) event->data.scalar.value) < 0) { flb_error("unable to add values to list"); return YAML_FAILURE; } @@ -1685,11 +2317,13 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, /* This is also the end of the plugin values mapping. * So we pop an additional state off the stack. */ - state = state_pop(ctx); + if (state->state == STATE_PLUGIN_VAL) { + state = state_pop(ctx); - if (state == NULL) { - flb_error("no state left"); - return YAML_FAILURE; + if (state == NULL) { + flb_error("no state left"); + return YAML_FAILURE; + } } break; default: @@ -2036,7 +2670,6 @@ static int state_create_section(struct flb_cf *conf, struct parser_state *state, } state->cf_section = flb_cf_section_create(conf, name, 0); - if (state->cf_section == NULL) { return -1; } @@ -2051,7 +2684,7 @@ static int state_create_group(struct flb_cf *conf, struct parser_state *state, c } state->cf_group = flb_cf_group_create(conf, state->cf_section, - "processors", strlen("processors")); + name, strlen(name)); if (state->cf_group == NULL) { return -1; @@ -2322,7 +2955,9 @@ struct flb_cf *flb_cf_yaml_create(struct flb_cf *conf, char *file_path, if (!conf) { return NULL; } - + flb_cf_set_origin_format(conf, FLB_CF_YAML); + } + else { flb_cf_set_origin_format(conf, FLB_CF_YAML); } diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index 2a6ab5f4601..92f92add099 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -119,7 +119,16 @@ struct flb_cf *flb_cf_create() mk_list_init(&ctx->parsers); mk_list_init(&ctx->multiline_parsers); - /* custom plugins */ + /* stream processors */ + mk_list_init(&ctx->stream_processors); + + /* external plugins (*.so) */ + mk_list_init(&ctx->plugins); + + /* upstream servers */ + mk_list_init(&ctx->upstream_servers); + + /* 'custom' type plugins */ mk_list_init(&ctx->customs); /* pipeline */ @@ -160,29 +169,38 @@ int flb_cf_set_origin_format(struct flb_cf *cf, int format) static enum section_type get_section_type(char *name, int len) { - if (strncasecmp(name, "SERVICE", len) == 0) { + if (strncasecmp(name, "service", len) == 0) { return FLB_CF_SERVICE; } - else if (strncasecmp(name, "PARSER", len) == 0) { + else if (strncasecmp(name, "parser", len) == 0) { return FLB_CF_PARSER; } - else if (strncasecmp(name, "MULTILINE_PARSER", len) == 0) { + else if (strncasecmp(name, "multiline_parser", len) == 0) { return FLB_CF_MULTILINE_PARSER; } - else if (strncasecmp(name, "CUSTOM", len) == 0 || - strncasecmp(name, "CUSTOMS", len) == 0) { + else if (strncasecmp(name, "stream_processor", len) == 0) { + return FLB_CF_STREAM_PROCESSOR; + } + else if (strncasecmp(name, "plugins", len) == 0) { + return FLB_CF_PLUGINS; + } + else if (strncasecmp(name, "upstream_servers", len) == 0) { + return FLB_CF_UPSTREAM_SERVERS; + } + else if (strncasecmp(name, "custom", len) == 0 || + strncasecmp(name, "customs", len) == 0) { return FLB_CF_CUSTOM; } - else if (strncasecmp(name, "INPUT", len) == 0 || - strncasecmp(name, "INPUTS", len) == 0) { + else if (strncasecmp(name, "input", len) == 0 || + strncasecmp(name, "inputs", len) == 0) { return FLB_CF_INPUT; } - else if (strncasecmp(name, "FILTER", len) == 0 || - strncasecmp(name, "FILTERS", len) == 0) { + else if (strncasecmp(name, "filter", len) == 0 || + strncasecmp(name, "filters", len) == 0) { return FLB_CF_FILTER; } - else if (strncasecmp(name, "OUTPUT", len) == 0 || - strncasecmp(name, "OUTPUTS", len) == 0) { + else if (strncasecmp(name, "output", len) == 0 || + strncasecmp(name, "outputs", len) == 0) { return FLB_CF_OUTPUT; } @@ -634,6 +652,15 @@ struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int else if (type == FLB_CF_MULTILINE_PARSER) { mk_list_add(&s->_head_section, &cf->multiline_parsers); } + else if (type == FLB_CF_STREAM_PROCESSOR) { + mk_list_add(&s->_head_section, &cf->stream_processors); + } + else if (type == FLB_CF_PLUGINS) { + mk_list_add(&s->_head_section, &cf->plugins); + } + else if (type == FLB_CF_UPSTREAM_SERVERS) { + mk_list_add(&s->_head_section, &cf->upstream_servers); + } else if (type == FLB_CF_CUSTOM) { mk_list_add(&s->_head_section, &cf->customs); } @@ -728,6 +755,12 @@ static char *section_type_str(int type) return "PARSER"; case FLB_CF_MULTILINE_PARSER: return "MULTILINE_PARSER"; + case FLB_CF_STREAM_PROCESSOR: + return "STREAM_PROCESSOR"; + case FLB_CF_PLUGINS: + return "PLUGINS"; + case FLB_CF_UPSTREAM_SERVERS: + return "UPSTREAM_SERVERS"; case FLB_CF_CUSTOM: return "CUSTOM"; case FLB_CF_INPUT: diff --git a/src/flb_config.c b/src/flb_config.c index 747d855cf08..32dc34b7e83 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -907,11 +907,21 @@ int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) /* Extra sanity checks */ if (strcasecmp(s->name, "parser") == 0 || strcasecmp(s->name, "multiline_parser") == 0) { - fprintf(stderr, - "Sections 'multiline_parser' and 'parser' are not valid in " - "the main configuration file. It belongs to \n" - "the 'parsers_file' configuration files.\n"); - return -1; + + /* + * Classic mode configuration don't allow parser or multiline_parser + * to be defined in the main configuration file. + */ + if (cf->format == FLB_CF_CLASSIC) { + fprintf(stderr, + "Sections 'multiline_parser' and 'parser' are not valid in " + "the main configuration file. It belongs to \n" + "the 'parsers_file' configuration files.\n"); + return -1; + } + else { + /* Yaml allow parsers definitions in any Yaml file, all good */ + } } } @@ -925,6 +935,21 @@ int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) } } + ret = flb_parser_load_parser_definitions("", cf, config); + if (ret == -1) { + return -1; + } + + ret = flb_parser_load_multiline_parser_definitions("", cf, config); + if (ret == -1) { + return -1; + } + + ret = flb_plugin_load_config_format(cf, config); + if (ret == -1) { + return -1; + } + ret = configure_plugins_type(config, cf, FLB_CF_CUSTOM); if (ret == -1) { return -1; diff --git a/src/flb_engine.c b/src/flb_engine.c index 911c1389a79..bedc28477c5 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -54,6 +54,7 @@ #include #include #include +#include #ifdef FLB_HAVE_METRICS #include @@ -799,6 +800,9 @@ int flb_engine_start(struct flb_config *config) return -1; } + /* Internals */ + flb_info("[simd ] %s", flb_simd_info()); + /* Init Metrics engine */ cmt_initialize(); flb_info("[cmetrics] version=%s", cmt_version()); diff --git a/src/flb_parser.c b/src/flb_parser.c index 8df7f56dfb5..149ab7f1b91 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -313,15 +313,15 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, p->time_frac_secs = (tmp + 2); } - /* - * Fall back to the system timezone - * if there is no zone parsed from the log. + /* + * Fall back to the system timezone + * if there is no zone parsed from the log. */ p->time_system_timezone = time_system_timezone; - /* - * Optional fixed timezone offset, only applied if - * not falling back to system timezone. + /* + * Optional fixed timezone offset, only applied if + * not falling back to system timezone. */ if (!p->time_system_timezone && time_offset) { diff = 0; @@ -481,9 +481,9 @@ static flb_sds_t get_parser_key(struct flb_config *config, return val; } -/* Config file: read 'parser' definitions */ -static int parser_conf_file(const char *cfg, struct flb_cf *cf, - struct flb_config *config) +/* Load each parser definition set in 'struct flb_cf *cf' */ +int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, + struct flb_config *config) { int i = 0; flb_sds_t name; @@ -541,7 +541,7 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, name, cfg); goto fconf_early_error; } - + /* skip_empty_values */ skip_empty = FLB_TRUE; tmp_str = get_parser_key(config, cf, s, "skip_empty_values"); @@ -605,7 +605,7 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, /* Create the parser context */ if (!flb_parser_create(name, format, regex, skip_empty, time_fmt, time_key, time_offset, time_keep, time_strict, - time_system_timezone, logfmt_no_bare_keys, types, types_len, + time_system_timezone, logfmt_no_bare_keys, types, types_len, decoders, config)) { goto fconf_error; } @@ -680,6 +680,17 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, return -1; } +static int multiline_rule_create(struct flb_ml_parser *ml_parser, + char *from_state, + char *regex_pattern, + char *to_state) +{ + int ret; + + ret = flb_ml_rule_create(ml_parser, from_state, regex_pattern, to_state, NULL); + return ret; +} + static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, struct flb_cf_section *section, struct flb_config *config) @@ -692,7 +703,47 @@ static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, struct flb_slist_entry *from_state; struct flb_slist_entry *regex_pattern; struct flb_slist_entry *tmp; + struct mk_list *g_head; + struct flb_cf_group *group; + struct cfl_variant *var_state; + struct cfl_variant *var_regex; + struct cfl_variant *var_next_state; + + /* Check if we have groups (coming from Yaml style config */ + mk_list_foreach(g_head, §ion->groups) { + /* Every group is a rule */ + group = cfl_list_entry(g_head, struct flb_cf_group, _head); + + var_state = cfl_kvlist_fetch(group->properties, "state"); + if (!var_state || var_state->type != CFL_VARIANT_STRING) { + flb_error("[multiline parser: %s] invalid 'state' key", ml_parser->name); + return -1; + } + + var_regex = cfl_kvlist_fetch(group->properties, "regex"); + if (!var_regex || var_regex->type != CFL_VARIANT_STRING) { + flb_error("[multiline parser: %s] invalid 'regex' key", ml_parser->name); + return -1; + } + var_next_state = cfl_kvlist_fetch(group->properties, "next_state"); + if (!var_next_state || var_next_state->type != CFL_VARIANT_STRING) { + flb_error("[multiline parser: %s] invalid 'next_state' key", ml_parser->name); + return -1; + } + + ret = multiline_rule_create(ml_parser, + var_state->data.as_string, + var_regex->data.as_string, + var_next_state->data.as_string); + + if (ret == -1) { + flb_error("[multiline parser: %s] error creating rule", ml_parser->name); + return -1; + } + } + + /* Multiline rules set by a Fluent Bit classic mode config */ cfl_list_foreach(head, §ion->properties->list) { entry = cfl_list_entry(head, struct cfl_kvpair, _head); @@ -705,7 +756,7 @@ static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, ret = flb_slist_split_tokens(&list, entry->val->data.as_string, 3); if (ret == -1) { flb_error("[multiline parser: %s] invalid section on key '%s'", - ml_parser->name, entry->key); + ml_parser->name, entry->key); return -1; } @@ -734,11 +785,10 @@ static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, return -1; } - ret = flb_ml_rule_create(ml_parser, - from_state->str, - regex_pattern->str, - to_state, - NULL); + ret = multiline_rule_create(ml_parser, + from_state->str, + regex_pattern->str, + to_state); if (ret == -1) { flb_error("[multiline parser: %s] error creating rule", ml_parser->name); @@ -762,8 +812,8 @@ static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, /* config file: read 'multiline_parser' sections */ -static int multiline_parser_conf_file(const char *cfg, struct flb_cf *cf, - struct flb_config *config) +int flb_parser_load_multiline_parser_definitions(const char *cfg, struct flb_cf *cf, + struct flb_config *config) { int ret; int type; @@ -781,6 +831,10 @@ static int multiline_parser_conf_file(const char *cfg, struct flb_cf *cf, struct flb_cf_section *s; struct flb_ml_parser *ml_parser; + /* + * debug content of cf: flb_cf_dump(cf); + */ + /* read all 'multiline_parser' sections */ mk_list_foreach(head, &cf->multiline_parsers) { ml_parser = NULL; @@ -946,15 +1000,15 @@ int flb_parser_conf_file(const char *file, struct flb_config *config) return -1; } - /* process 'parser' sections */ - ret = parser_conf_file(cfg, cf, config); + /* load the parser definitions */ + ret = flb_parser_load_parser_definitions(cfg, cf, config); if (ret == -1) { flb_cf_destroy(cf); return -1; } /* processs 'multiline_parser' sections */ - ret = multiline_parser_conf_file(cfg, cf, config); + ret = flb_parser_load_multiline_parser_definitions(cfg, cf, config); if (ret == -1) { flb_cf_destroy(cf); return -1; diff --git a/src/flb_plugin.c b/src/flb_plugin.c index a26bd013576..d55fced3287 100644 --- a/src/flb_plugin.c +++ b/src/flb_plugin.c @@ -353,6 +353,33 @@ int flb_plugin_load_router(char *path, struct flb_config *config) return 0; } +int flb_plugin_load_config_format(struct flb_cf *cf, struct flb_config *config) +{ + int ret; + struct mk_list *head; + struct cfl_list *head_e; + struct flb_cf_section *section; + struct cfl_kvpair *entry; + + /* read all 'plugins' sections */ + mk_list_foreach(head, &cf->plugins) { + section = mk_list_entry(head, struct flb_cf_section, _head_section); + + cfl_list_foreach(head_e, §ion->properties->list) { + entry = cfl_list_entry(head_e, struct cfl_kvpair, _head); + + /* Load plugin with router function */ + ret = flb_plugin_load_router(entry->key, config); + if (ret == -1) { + flb_cf_destroy(cf); + return -1; + } + } + } + + return 0; +} + /* Load plugins from a configuration file */ int flb_plugin_load_config_file(const char *file, struct flb_config *config) { @@ -395,7 +422,17 @@ int flb_plugin_load_config_file(const char *file, struct flb_config *config) return -1; } - /* read all 'plugins' sections */ + /* + * pass to the config_format loader also in case some Yaml have been included in + * the service section through the option 'plugins_file' + */ + ret = flb_plugin_load_config_format(cf, config); + if (ret == -1) { + flb_cf_destroy(cf); + return -1; + } + + /* (classic mode) read all 'plugins' sections */ mk_list_foreach(head, &cf->sections) { section = mk_list_entry(head, struct flb_cf_section, _head); if (strcasecmp(section->name, "plugins") != 0) { diff --git a/src/flb_processor.c b/src/flb_processor.c index 00c892491f4..a9d3dce5705 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -660,11 +660,13 @@ int flb_processor_run(struct flb_processor *proc, return -1; } - if (cur_buf != data) { + if (cur_buf != data && cur_buf != tmp_buf) { cmt_destroy(cur_buf); } - cur_buf = (void *)tmp_buf; + if (tmp_buf != NULL) { + cur_buf = tmp_buf; + } } } else if (type == FLB_PROCESSOR_TRACES) { diff --git a/src/flb_upstream_ha.c b/src/flb_upstream_ha.c index 62c6b3db956..d01e55ad5f3 100644 --- a/src/flb_upstream_ha.c +++ b/src/flb_upstream_ha.c @@ -363,10 +363,13 @@ struct flb_upstream_ha *flb_upstream_ha_from_file(const char *file, char path[PATH_MAX + 1]; struct stat st; struct mk_list *head; + struct mk_list *g_head; struct flb_upstream_ha *ups; struct flb_upstream_node *node; struct flb_cf *cf = NULL; struct flb_cf_section *section; + struct flb_cf_group *group; + struct flb_cf_section *node_section; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -394,50 +397,110 @@ struct flb_upstream_ha *flb_upstream_ha_from_file(const char *file, return NULL; } - /* 'upstream' sections are under enum section_type FLB_CF_OTHER */ - section = flb_cf_section_get_by_name(cf, "upstream"); - if (!section) { - flb_error("[upstream_ha] section name 'upstream' could not be found"); - flb_cf_destroy(cf); - return NULL; - } - - /* upstream name */ - tmp = flb_cf_section_property_get_string(cf, section, "name"); - if (!tmp) { - flb_error("[upstream_ha] missing name for upstream at %s", cfg); - flb_cf_destroy(cf); - return NULL; - } - - ups = flb_upstream_ha_create(tmp); - flb_sds_destroy(tmp); - if (!ups) { - flb_error("[upstream_ha] cannot create context"); - flb_cf_destroy(cf); - return NULL; - } + if (cf->format == FLB_CF_FLUENTBIT) { + /* 'upstream' sections are under enum section_type FLB_CF_OTHER */ + section = flb_cf_section_get_by_name(cf, "upstream"); + if (!section) { + flb_error("[upstream_ha] section name 'upstream' could not be found"); + flb_cf_destroy(cf); + return NULL; + } - /* 'node' sections */ - mk_list_foreach(head, &cf->sections) { - section = mk_list_entry(head, struct flb_cf_section, _head); - if (strcasecmp(section->name, "node") != 0) { - continue; + /* upstream name */ + tmp = flb_cf_section_property_get_string(cf, section, "name"); + if (!tmp) { + flb_error("[upstream_ha] missing name for upstream at %s", cfg); + flb_cf_destroy(cf); + return NULL; } - /* Read section info and create a Node context */ - node = create_node(c, cf, section, config); - if (!node) { - flb_error("[upstream_ha] cannot register node on upstream '%s'", - tmp); - flb_upstream_ha_destroy(ups); + ups = flb_upstream_ha_create(tmp); + flb_sds_destroy(tmp); + if (!ups) { + flb_error("[upstream_ha] cannot create context"); flb_cf_destroy(cf); return NULL; } - flb_upstream_ha_node_add(ups, node); - c++; + /* 'node' sections */ + mk_list_foreach(head, &cf->sections) { + section = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(section->name, "node") != 0) { + continue; + } + + /* Read section info and create a Node context */ + node = create_node(c, cf, section, config); + if (!node) { + flb_error("[upstream_ha] cannot register node on upstream '%s'", + tmp); + flb_upstream_ha_destroy(ups); + flb_cf_destroy(cf); + return NULL; + } + + flb_upstream_ha_node_add(ups, node); + c++; + } } +#ifdef FLB_HAVE_LIBYAML + else if (cf->format == FLB_CF_YAML) { + mk_list_foreach(head, &cf->upstream_servers) { + section = mk_list_entry(head, struct flb_cf_section, _head_section); + + /* upstream name */ + tmp = flb_cf_section_property_get_string(cf, section, "name"); + if (!tmp) { + flb_error("[upstream_ha] missing name for upstream at %s", cfg); + flb_cf_destroy(cf); + return NULL; + } + + ups = flb_upstream_ha_create(tmp); + flb_sds_destroy(tmp); + if (!ups) { + flb_error("[upstream_ha] cannot create context"); + flb_cf_destroy(cf); + return NULL; + } + + /* iterate nodes (groups) */ + mk_list_foreach(g_head, §ion->groups) { + group = mk_list_entry(g_head, struct flb_cf_group, _head); + + /* + * create temporary node section: the node creation function needs a section, + * which is not the same as the group but similar: we just map the name and + * properties. + */ + node_section = flb_calloc(1, sizeof(struct flb_cf_section)); + if (!node_section) { + flb_errno(); + flb_upstream_ha_destroy(ups); + flb_cf_destroy(cf); + return NULL; + } + node_section->name = group->name; + node_section->properties = group->properties; + + /* Read section info and create a Node context */ + node = create_node(c, cf, node_section, config); + if (!node) { + flb_error("[upstream_ha] cannot register node on upstream '%s'", + tmp); + flb_upstream_ha_destroy(ups); + flb_cf_destroy(cf); + flb_free(node_section); + return NULL; + } + flb_free(node_section); + + flb_upstream_ha_node_add(ups, node); + c++; + } + } + } +#endif if (c == 0) { flb_error("[upstream_ha] no nodes defined"); diff --git a/src/flb_utils.c b/src/flb_utils.c index 010254f23c5..211632260e3 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -35,6 +35,7 @@ #include #include #include +#include #ifdef FLB_HAVE_AWS_ERROR_REPORTER #include @@ -577,7 +578,7 @@ int64_t flb_utils_size_to_bytes(const char *size) if (tmp[0] == 'K') { /* set upper bound (2**64/KB)/2 to avoid overflows */ - if (val >= 9223372036854775 || val <= -9223372036854774) + if (val >= 9223372036854775.0 || val <= -9223372036854774.0) { return -1; } @@ -756,229 +757,308 @@ static inline void encoded_to_buf(char *out, const char *in, int len) } } +/* Structure to hold escape sequence */ +struct escape_seq { + const char *seq; +}; + +/* Lookup table for escape sequences */ +static const struct escape_seq json_escape_table[128] = { + ['\"'] = {"\\\""}, + ['\\'] = {"\\\\"}, + ['\n'] = {"\\n"}, + ['\r'] = {"\\r"}, + ['\t'] = {"\\t"}, + ['\b'] = {"\\b"}, + ['\f'] = {"\\f"}, + [0x00] = {"\\u0000"}, [0x01] = {"\\u0001"}, [0x02] = {"\\u0002"}, [0x03] = {"\\u0003"}, + [0x04] = {"\\u0004"}, [0x05] = {"\\u0005"}, [0x06] = {"\\u0006"}, [0x07] = {"\\u0007"}, + [0x0B] = {"\\u000b"}, [0x0E] = {"\\u000e"}, [0x0F] = {"\\u000f"}, + [0x10] = {"\\u0010"}, [0x11] = {"\\u0011"}, [0x12] = {"\\u0012"}, [0x13] = {"\\u0013"}, + [0x14] = {"\\u0014"}, [0x15] = {"\\u0015"}, [0x16] = {"\\u0016"}, [0x17] = {"\\u0017"}, + [0x18] = {"\\u0018"}, [0x19] = {"\\u0019"}, [0x1A] = {"\\u001a"}, [0x1B] = {"\\u001b"}, + [0x1C] = {"\\u001c"}, [0x1D] = {"\\u001d"}, [0x1E] = {"\\u001e"}, [0x1F] = {"\\u001f"}, + [0x7F] = {"\\u007f"} +}; + /* * Write string pointed by 'str' to the destination buffer 'buf'. It's make sure * to escape special characters and convert utf-8 byte characters to string * representation. */ -int flb_utils_write_str(char *buf, int *off, size_t size, - const char *str, size_t str_len) + +int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_t str_len) { - int i; - int b; - int ret; - int written = 0; - int required; - int len; - int hex_bytes; - int is_valid; - int utf_sequence_number; - int utf_sequence_length; - uint32_t codepoint; - uint32_t state = 0; + int i, b, ret, len, hex_bytes, utf_sequence_length, utf_sequence_number; + int is_valid, copypos = 0, vlen; + uint32_t codepoint, state = 0; char tmp[16]; size_t available; uint32_t c; char *p; uint8_t *s; + off_t offset = 0; + + available = size - *off; - available = (size - *off); - required = str_len; - if (available <= required) { + /* Ensure we have some minimum space in the buffer */ + if (available < str_len) { return FLB_FALSE; } p = buf + *off; - for (i = 0; i < str_len; i++) { - if ((available - written) < 2) { - return FLB_FALSE; - } - c = (uint32_t) str[i]; - if (c == '\"') { - *p++ = '\\'; - *p++ = '\"'; - } - else if (c == '\\') { - *p++ = '\\'; - *p++ = '\\'; - } - else if (c == '\n') { - *p++ = '\\'; - *p++ = 'n'; - } - else if (c == '\r') { - *p++ = '\\'; - *p++ = 'r'; - } - else if (c == '\t') { - *p++ = '\\'; - *p++ = 't'; - } - else if (c == '\b') { - *p++ = '\\'; - *p++ = 'b'; - } - else if (c == '\f') { - *p++ = '\\'; - *p++ = 'f'; - } - else if (c < 32 || c == 0x7f) { - if ((available - written) < 6) { - return FLB_FALSE; - } - len = snprintf(tmp, sizeof(tmp) - 1, "\\u%.4hhx", (unsigned char) c); - if ((available - written) < len) { - return FLB_FALSE; + /* align length to the nearest multiple of the vector size for safe SIMD processing */ + vlen = str_len & ~(sizeof(flb_vector8) - 1); + for (i = 0;;) { + /* SIMD optimization: Process chunk of input string */ + for (; i < vlen; i += sizeof(flb_vector8)) { + flb_vector8 chunk; + flb_vector8_load(&chunk, (const uint8_t *)&str[i]); + + /* + * Look for the special characters we are interested in, + * if they are found we break the loop and escape them + * in a char-by-char basis. Otherwise the do a bulk copy + */ + if (flb_vector8_has_le(chunk, (unsigned char) 0x1F) || + flb_vector8_has(chunk, (unsigned char) '"') || + flb_vector8_has(chunk, (unsigned char) '\\')) { + break; } - encoded_to_buf(p, tmp, len); - p += len; } - else if (c >= 0x80 && c <= 0xFFFF) { - hex_bytes = flb_utf8_len(str + i); - if (available - written < 6) { + + /* Copy the chunk processed so far */ + if (copypos < i) { + /* check if we have enough space */ + if (available < i - copypos) { return FLB_FALSE; } - if (i + hex_bytes > str_len) { - break; /* skip truncated UTF-8 */ + /* copy and adjust pointers */ + memcpy(p, &str[copypos], i - copypos); + p += i - copypos; + offset += i - copypos; + available -= (i - copypos); + copypos = i; + } + + /* Process remaining characters one by one */ + for (b = 0; b < sizeof(flb_vector8); b++) { + if (i >= str_len) { + /* all characters has been processed */ + goto done; } - state = FLB_UTF8_ACCEPT; - codepoint = 0; + c = (uint32_t) str[i]; - for (b = 0; b < hex_bytes; b++) { - s = (unsigned char *) str + i + b; - ret = flb_utf8_decode(&state, &codepoint, *s); - if (ret == 0) { - break; + /* Use lookup table for escaping known sequences */ + if (c < 128 && json_escape_table[c].seq) { + /* + * All characters in the table have a lenght of 2 or 6 bytes, + * just check if the second byte starts with 'u' so we know + * it's unicode and needs 6 bytes of space. + */ + if (json_escape_table[c].seq[1] == 'u') { + len = 6; + } + else { + len = 2; } - } - if (state != FLB_UTF8_ACCEPT) { - /* Invalid UTF-8 hex, just skip utf-8 bytes */ - flb_warn("[pack] invalid UTF-8 bytes found, skipping bytes"); - } - else { - len = snprintf(tmp, sizeof(tmp) - 1, "\\u%.4x", codepoint); - if ((available - written) < len) { + /* check if we have anough space */ + if (available < len) { return FLB_FALSE; } - encoded_to_buf(p, tmp, len); + + /* copy the escape sequence */ + memcpy(p, json_escape_table[c].seq, len); p += len; + offset += len; + available -= len; } - i += (hex_bytes - 1); - } - else if (c > 0xFFFF) { - utf_sequence_length = flb_utf8_len(str + i); + /* Handle UTF-8 sequences from 0x80 to 0xFFFF */ + else if (c >= 0x80 && c <= 0xFFFF) { + hex_bytes = flb_utf8_len(&str[i]); + + /* Handle invalid or truncated sequence */ + if (hex_bytes == 0 || i + hex_bytes > str_len) { + /* check for the minimum space required */ + if (available < 3) { + return FLB_FALSE; + } + + /* insert replacement character (U+FFFD) */ + p[0] = 0xEF; + p[1] = 0xBF; + p[2] = 0xBD; + p += 3; + offset += 3; + available -= 3; + + /* skip the original byte */ + i++; + continue; + } + + /* decode UTF-8 sequence */ + state = FLB_UTF8_ACCEPT; + codepoint = 0; + + for (b = 0; b < hex_bytes; b++) { + s = (unsigned char *) &str[i + b]; + ret = flb_utf8_decode(&state, &codepoint, *s); + if (ret == 0) { + break; + } + } + + if (state != FLB_UTF8_ACCEPT) { + flb_warn("[pack] Invalid UTF-8 bytes found, skipping."); + } + else { + + len = snprintf(tmp, sizeof(tmp), "\\u%.4x", codepoint); + if (available < len) { + return FLB_FALSE; // Not enough space + } + memcpy(p, tmp, len); + p += len; + offset += len; + available -= len; + } - if (i + utf_sequence_length > str_len) { - break; /* skip truncated UTF-8 */ + i += hex_bytes; } + /* Handle sequences beyond 0xFFFF */ + else if (c > 0xFFFF) { + utf_sequence_length = flb_utf8_len(str + i); - is_valid = FLB_TRUE; - for (utf_sequence_number = 0; utf_sequence_number < utf_sequence_length; - utf_sequence_number++) { - /* Leading characters must start with bits 11 */ - if (utf_sequence_number == 0 && ((str[i] & 0xC0) != 0xC0)) { - /* Invalid unicode character. replace */ - flb_debug("[pack] unexpected UTF-8 leading byte, " - "substituting character with replacement character"); - tmp[utf_sequence_number] = str[i]; - ++i; /* Consume invalid leading byte */ - utf_sequence_length = utf_sequence_number + 1; - is_valid = FLB_FALSE; + /* skip truncated UTF-8 ? */ + if (i + utf_sequence_length > str_len) { + i++; break; } - /* Trailing characters must start with bits 10 */ - else if (utf_sequence_number > 0 && ((str[i] & 0xC0) != 0x80)) { - /* Invalid unicode character. replace */ - flb_debug("[pack] unexpected UTF-8 continuation byte, " - "substituting character with replacement character"); - /* This byte, i, is the start of the next unicode character */ - utf_sequence_length = utf_sequence_number; - is_valid = FLB_FALSE; - break; + + is_valid = FLB_TRUE; + for (utf_sequence_number = 0; utf_sequence_number < utf_sequence_length; utf_sequence_number++) { + /* Leading characters must start with bits 11 */ + if (utf_sequence_number == 0 && ((str[i] & 0xC0) != 0xC0)) { + /* Invalid unicode character. replace */ + flb_debug("[pack] unexpected UTF-8 leading byte, " + "substituting character with replacement character"); + tmp[utf_sequence_number] = str[i]; + i++; /* Consume invalid leading byte */ + utf_sequence_length = utf_sequence_number + 1; + is_valid = FLB_FALSE; + break; + } + /* Trailing characters must start with bits 10 */ + else if (utf_sequence_number > 0 && ((str[i] & 0xC0) != 0x80)) { + /* Invalid unicode character. replace */ + flb_debug("[pack] unexpected UTF-8 continuation byte, " + "substituting character with replacement character"); + /* This byte, i, is the start of the next unicode character */ + utf_sequence_length = utf_sequence_number; + is_valid = FLB_FALSE; + break; + } + + tmp[utf_sequence_number] = str[i]; + ++i; } - tmp[utf_sequence_number] = str[i]; - ++i; - } - --i; + --i; - if (is_valid) { - if (available - written < utf_sequence_length) { - return FLB_FALSE; + if (is_valid) { + if (available < utf_sequence_length) { + return FLB_FALSE; // Not enough space + } + + encoded_to_buf(p, tmp, utf_sequence_length); + p += utf_sequence_length; + offset += utf_sequence_length; + available -= utf_sequence_length; } + else { + if (available < utf_sequence_length * 3) { + return FLB_FALSE; + } - encoded_to_buf(p, tmp, utf_sequence_length); - p += utf_sequence_length; + /* + * Utf-8 sequence is invalid. Map fragments to private use area + * codepoints in range: + * 0x00 to + * 0xFF + */ + for (b = 0; b < utf_sequence_length; ++b) { + /* + * Utf-8 private block invalid hex mapping. Format unicode charpoint + * in the following format: + * + * +--------+--------+--------+ + * |1110PPPP|10PPPPHH|10HHHHHH| + * +--------+--------+--------+ + * + * Where: + * P is FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR bits (1 byte) + * H is Utf-8 fragment hex bits (1 byte) + * 1 is bit 1 + * 0 is bit 0 + */ + + /* unicode codepoint start */ + *p = 0xE0; + + /* print unicode private block header first 4 bits */ + *p |= FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR >> 4; + ++p; + + /* unicode codepoint middle */ + *p = 0x80; + + /* print end of unicode private block header last 4 bits */ + *p |= ((FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR << 2) & 0x3f); + + /* print hex fragment first 2 bits */ + *p |= (tmp[b] >> 6) & 0x03; + ++p; + + /* unicode codepoint middle */ + *p = 0x80; + + /* print hex fragment last 6 bits */ + *p |= tmp[b] & 0x3f; + ++p; + + offset += 3; + available -= 3; + } + } } else { - if (available - written < utf_sequence_length * 3) { + if (available < 1) { + /* no space for a single byte */ return FLB_FALSE; } - - /* - * Utf-8 sequence is invalid. Map fragments to private use area - * codepoints in range: - * 0x00 to - * 0xFF - */ - for (b = 0; b < utf_sequence_length; ++b) { - /* - * Utf-8 private block invalid hex mapping. Format unicode charpoint - * in the following format: - * - * +--------+--------+--------+ - * |1110PPPP|10PPPPHH|10HHHHHH| - * +--------+--------+--------+ - * - * Where: - * P is FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR bits (1 byte) - * H is Utf-8 fragment hex bits (1 byte) - * 1 is bit 1 - * 0 is bit 0 - */ - - /* unicode codepoint start */ - *p = 0xE0; - - /* print unicode private block header first 4 bits */ - *p |= FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR >> 4; - ++p; - - /* unicode codepoint middle */ - *p = 0x80; - - /* print end of unicode private block header last 4 bits */ - *p |= ((FLB_UTILS_FRAGMENT_PRIVATE_BLOCK_DESCRIPTOR << 2) & 0x3f); - - /* print hex fragment first 2 bits */ - *p |= (tmp[b] >> 6) & 0x03; - ++p; - - /* unicode codepoint middle */ - *p = 0x80; - - /* print hex fragment last 6 bits */ - *p |= tmp[b] & 0x3f; - ++p; - } + *p++ = c; + offset++; + available--; } + + i++; } - else { - *p++ = c; - } - written = (p - (buf + *off)); + + copypos = i; } - *off += written; +done: + /* update the buffer offset */ + *off += offset; return FLB_TRUE; } - int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t *out_size) { int ret; diff --git a/src/stream_processor/flb_sp.c b/src/stream_processor/flb_sp.c index 96aa508d9d1..07a19abddc5 100644 --- a/src/stream_processor/flb_sp.c +++ b/src/stream_processor/flb_sp.c @@ -96,10 +96,19 @@ static int sp_config_file(struct flb_config *config, struct flb_sp *sp, return -1; } - /* Read all 'stream_task' sections */ + /* + * Note on reading the sections + * ---------------------------- + * Classic mode configuration looks for [STREAM_TASK], while the + * new Yaml parser expects the section names to be stream_processor. + * + * On Yaml mode, each pair of "name/exec" is set as an independent section, + * so the adjusted code below works for both type of files. + */ mk_list_foreach(head, &cf->sections) { section = mk_list_entry(head, struct flb_cf_section, _head); - if (strcasecmp(section->name, "stream_task") != 0) { + if (strcasecmp(section->name, "stream_task") != 0 && + strcasecmp(section->name, "stream_processor") != 0) { continue; } @@ -689,10 +698,14 @@ struct flb_sp *flb_sp_create(struct flb_config *config) int i = 0; int ret; char buf[32]; + char *task_name; + char *task_exec; struct mk_list *head; struct flb_sp *sp; struct flb_slist_entry *e; struct flb_sp_task *task; + struct cfl_variant *var; + struct flb_cf_section *section; /* Allocate context */ sp = flb_malloc(sizeof(struct flb_sp)); @@ -714,6 +727,35 @@ struct flb_sp *flb_sp_create(struct flb_config *config) } } + /* register stream processor tasks registered through Yaml config */ + if (config->cf_main) { + mk_list_foreach(head, &config->cf_main->stream_processors) { + section = mk_list_entry(head, struct flb_cf_section, _head_section); + + /* task name */ + var = cfl_kvlist_fetch(section->properties, "name"); + if (!var || var->type != CFL_VARIANT_STRING) { + flb_error("[sp] missing 'name' property in stream_processor section"); + continue; + } + task_name = var->data.as_string; + + /* task exec/query */ + var = cfl_kvlist_fetch(section->properties, "exec"); + if (!var || var->type != CFL_VARIANT_STRING) { + flb_error("[sp] missing 'exec' property in stream_processor section"); + continue; + } + task_exec = var->data.as_string; + + /* create task */ + task = flb_sp_task_create(sp, task_name, task_exec); + if (!task) { + continue; + } + } + } + /* Lookup configuration file if any */ if (config->stream_processor_file) { ret = sp_config_file(config, sp, config->stream_processor_file); diff --git a/src/tls/openssl.c b/src/tls/openssl.c index 38bba04e4f0..043a065efc4 100644 --- a/src/tls/openssl.c +++ b/src/tls/openssl.c @@ -241,18 +241,28 @@ static int windows_load_system_certificates(struct tls_context *ctx) { int ret; HANDLE win_store; + unsigned long err; PCCERT_CONTEXT win_cert = NULL; const unsigned char *win_cert_data; X509_STORE *ossl_store = SSL_CTX_get_cert_store(ctx->ctx); X509 *ossl_cert; + /* Check if OpenSSL certificate store is available */ + if (!ossl_store) { + flb_error("[tls] failed to retrieve openssl certificate store."); + return -1; + } + + /* Open the Windows system certificate store */ win_store = CertOpenSystemStoreA(0, "Root"); if (win_store == NULL) { - flb_error("[tls] Cannot open cert store: %i", GetLastError()); + flb_error("[tls] cannot open windows certificate store: %lu", GetLastError()); return -1; } - while (win_cert = CertEnumCertificatesInStore(win_store, win_cert)) { + /* Iterate over certificates in the store */ + while ((win_cert = CertEnumCertificatesInStore(win_store, win_cert)) != NULL) { + /* Check if the certificate is encoded in ASN.1 DER format */ if (win_cert->dwCertEncodingType & X509_ASN_ENCODING) { /* * Decode the certificate into X509 struct. @@ -262,25 +272,42 @@ static int windows_load_system_certificates(struct tls_context *ctx) */ win_cert_data = win_cert->pbCertEncoded; ossl_cert = d2i_X509(NULL, &win_cert_data, win_cert->cbCertEncoded); + if (!ossl_cert) { - flb_debug("[tls] Cannot parse a certificate. skipping..."); + flb_debug("[tls] cannot parse a certificate, error code: %lu, skipping...", ERR_get_error()); continue; } /* Add X509 struct to the openssl cert store */ ret = X509_STORE_add_cert(ossl_store, ossl_cert); if (!ret) { - flb_warn("[tls] Failed to add a certificate to the store: %lu: %s", - ERR_get_error(), ERR_error_string(ERR_get_error(), NULL)); + err = ERR_get_error(); + if (err == X509_R_CERT_ALREADY_IN_HASH_TABLE) { + flb_debug("[tls] certificate already exists in the store, skipping."); + } + else { + flb_warn("[tls] failed to add certificate to openssl store. error code: %lu - %s", + err, ERR_error_string(err, NULL)); + } } X509_free(ossl_cert); } } + /* Check for errors during enumeration */ + if (GetLastError() != CRYPT_E_NOT_FOUND) { + flb_error("[tls] error occurred while enumerating certificates: %lu", GetLastError()); + CertCloseStore(win_store, 0); + return -1; + } + + /* Close the Windows system certificate store */ if (!CertCloseStore(win_store, 0)) { - flb_error("[tls] Cannot close cert store: %i", GetLastError()); + flb_error("[tls] cannot close windows certificate store: %lu", GetLastError()); return -1; } + + flb_debug("[tls] successfully loaded certificates from windows system store."); return 0; } #endif @@ -684,6 +711,7 @@ static int tls_net_handshake(struct flb_tls *tls, char err_buf[256]; struct tls_session *session = ptr_session; struct tls_context *ctx; + const char *x509_err; ctx = session->parent; pthread_mutex_lock(&ctx->mutex); @@ -743,8 +771,9 @@ static int tls_net_handshake(struct flb_tls *tls, if (ret == 0) { ssl_code = SSL_get_verify_result(session->ssl); if (ssl_code != X509_V_OK) { - flb_error("[tls] error: unexpected EOF with reason: %s", - ERR_reason_error_string(ERR_get_error())); + /* Refer to: https://x509errors.org/ */ + x509_err = X509_verify_cert_error_string(ssl_code); + flb_error("[tls] certificate verification failed, reason: %s (X509 code: %ld)", x509_err, ssl_code); } else { flb_error("[tls] error: unexpected EOF"); diff --git a/tests/internal/config_format_yaml.c b/tests/internal/config_format_yaml.c index f34bdb273d4..304dec9393c 100644 --- a/tests/internal/config_format_yaml.c +++ b/tests/internal/config_format_yaml.c @@ -20,6 +20,17 @@ #define FLB_000 FLB_TESTS_CONF_PATH "/fluent-bit.yaml" #define FLB_001 FLB_TESTS_CONF_PATH "/issue_7559.yaml" #define FLB_002 FLB_TESTS_CONF_PATH "/processors.yaml" + +#ifdef _WIN32 +#define FLB_003 FLB_TESTS_CONF_PATH "\\parsers_and_multiline_parsers.yaml" +#else +#define FLB_003 FLB_TESTS_CONF_PATH "/parsers_and_multiline_parsers.yaml" +#endif + +#define FLB_004 FLB_TESTS_CONF_PATH "/stream_processor.yaml" +#define FLB_005 FLB_TESTS_CONF_PATH "/plugins.yaml" +#define FLB_006 FLB_TESTS_CONF_PATH "/upstream.yaml" + #define FLB_000_WIN FLB_TESTS_CONF_PATH "\\fluent-bit-windows.yaml" #define FLB_BROKEN_PLUGIN_VARIANT FLB_TESTS_CONF_PATH "/broken_plugin_variant.yaml" @@ -289,7 +300,7 @@ static void test_parser_conf() /* Total number of inputs */ if(!TEST_CHECK(mk_list_size(&config->parsers) == cnt+1)) { - TEST_MSG("Section number error. Got=%d expect=%d", + TEST_MSG("Section number error. Got=%d expect=%d", mk_list_size(&config->parsers), cnt+1); } @@ -455,6 +466,373 @@ static void test_processors() flb_cf_destroy(cf); } +static void test_parsers_and_multiline_parsers() +{ + int idx = 0; + flb_sds_t str; + struct mk_list *head; + struct mk_list *rule_head; + struct flb_cf *cf; + struct flb_cf_section *s; + struct flb_cf_group *g; + struct cfl_variant *v; + struct cfl_variant *tmp; + + cf = flb_cf_yaml_create(NULL, FLB_003, NULL, 0); + TEST_CHECK(cf != NULL); + if (!cf) { + exit(EXIT_FAILURE); + } + + /* Total number of sections */ + TEST_CHECK(mk_list_size(&cf->sections) == 8); + + /* Check number sections per list */ + TEST_CHECK(mk_list_size(&cf->parsers) == 3); + TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 2); + TEST_CHECK(mk_list_size(&cf->customs) == 0); + TEST_CHECK(mk_list_size(&cf->inputs) == 1); + TEST_CHECK(mk_list_size(&cf->filters) == 0); + TEST_CHECK(mk_list_size(&cf->outputs) == 1); + TEST_CHECK(mk_list_size(&cf->others) == 0); + + /* check parsers */ + idx = 0; + mk_list_foreach(head, &cf->parsers) { + s = mk_list_entry(head, struct flb_cf_section, _head_section); + switch (idx) { + case 0: + v = flb_cf_section_property_get(cf, s, "name"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "json-2") == 0); + break; + case 1: + v = flb_cf_section_property_get(cf, s, "name"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "json") == 0); + break; + + case 2: + v = flb_cf_section_property_get(cf, s, "name"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "docker") == 0); + break; + } + idx++; + } + + /* check multiline parsers */ + idx = 0; + head = NULL; + mk_list_foreach(head, &cf->multiline_parsers) { + s = mk_list_entry(head, struct flb_cf_section, _head_section); + str = flb_cf_section_property_get_string(cf, s, "name"); + + switch (idx) { + case 0: + TEST_CHECK(strcmp(str, "exception_test-2") == 0); + break; + case 1: + TEST_CHECK(strcmp(str, "exception_test") == 0); + break; + }; + flb_sds_destroy(str); + + /* check rules (groups) */ + TEST_CHECK(mk_list_size(&s->groups) == 2); + + idx = 0; + mk_list_foreach(rule_head, &s->groups) { + g = mk_list_entry(rule_head, struct flb_cf_group, _head); + TEST_CHECK(strcmp(g->name, "rule") == 0); + + if (idx == 0) { + /* get initial state "start_state" */ + tmp = cfl_kvlist_fetch(g->properties, "state"); + TEST_CHECK(tmp != NULL); + + TEST_CHECK(tmp->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(tmp->data.as_string, "start_state") == 0); + } + else if (idx == 1) { + /* get initial state "start_state" */ + tmp = cfl_kvlist_fetch(g->properties, "state"); + TEST_CHECK(tmp != NULL); + + TEST_CHECK(tmp->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(tmp->data.as_string, "cont") == 0); + } + idx++; + } + } + + flb_cf_destroy(cf); +} + +static void test_stream_processor() +{ + int idx = 0; + struct mk_list *head; + struct flb_cf *cf; + struct flb_cf_section *s; + struct cfl_variant *v; + + cf = flb_cf_yaml_create(NULL, FLB_004, NULL, 0); + TEST_CHECK(cf != NULL); + if (!cf) { + exit(EXIT_FAILURE); + } + + /* Total number of sections */ + TEST_CHECK(mk_list_size(&cf->sections) == 5); + + /* Check number sections per list */ + TEST_CHECK(mk_list_size(&cf->parsers) == 0); + TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 0); + TEST_CHECK(mk_list_size(&cf->customs) == 0); + TEST_CHECK(mk_list_size(&cf->inputs) == 1); + TEST_CHECK(mk_list_size(&cf->filters) == 0); + TEST_CHECK(mk_list_size(&cf->outputs) == 1); + TEST_CHECK(mk_list_size(&cf->others) == 0); + + /* check others */ + idx = 0; + mk_list_foreach(head, &cf->stream_processors) { + s = mk_list_entry(head, struct flb_cf_section, _head_section); + + switch (idx) { + case 0: + v = flb_cf_section_property_get(cf, s, "name"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "create_results") == 0); + + v = flb_cf_section_property_get(cf, s, "exec"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strncmp(v->data.as_string, "CREATE STREAM results", 21) == 0); + break; + case 1: + v = flb_cf_section_property_get(cf, s, "name"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "select_results") == 0); + + v = flb_cf_section_property_get(cf, s, "exec"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strncmp(v->data.as_string, "SELECT * FROM", 13) == 0); + break; + }; + idx++; + + /* check groups */ + TEST_CHECK(mk_list_size(&s->groups) == 0); + } + + flb_cf_destroy(cf); +} + +static void test_plugins() +{ + int idx = 0; + struct mk_list *head; + struct flb_cf *cf; + struct flb_cf_section *s; + + struct cfl_kvpair *path; + struct cfl_list *path_head; + + cf = flb_cf_yaml_create(NULL, FLB_005, NULL, 0); + TEST_CHECK(cf != NULL); + if (!cf) { + exit(EXIT_FAILURE); + } + + /* Total number of sections */ + TEST_CHECK(mk_list_size(&cf->sections) == 4); + + /* Check number sections per list */ + TEST_CHECK(mk_list_size(&cf->plugins) == 1); + TEST_CHECK(mk_list_size(&cf->parsers) == 0); + TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 0); + TEST_CHECK(mk_list_size(&cf->customs) == 0); + TEST_CHECK(mk_list_size(&cf->inputs) == 1); + TEST_CHECK(mk_list_size(&cf->filters) == 0); + TEST_CHECK(mk_list_size(&cf->outputs) == 1); + TEST_CHECK(mk_list_size(&cf->others) == 0); + + + mk_list_foreach(head, &cf->plugins) { + s = mk_list_entry(head, struct flb_cf_section, _head_section); + + idx = 0; + cfl_list_foreach(path_head, &s->properties->list) { + path = cfl_list_entry(path_head, struct cfl_kvpair, _head); + + switch (idx) { + case 0: + TEST_CHECK(strcmp(path->key, "/path/to/out_gstdout.so") == 0); + break; + case 1: + TEST_CHECK(strcmp(path->key, "/path/to/out_fluent.so") == 0); + break; + }; + idx++; + } + } + + flb_cf_destroy(cf); +} + +static void test_upstream_servers() +{ + int idx = 0; + int g_idx = 0; + struct mk_list *head; + struct mk_list *g_head; + struct flb_cf *cf; + struct flb_cf_section *s; + struct cfl_variant *v; + struct flb_cf_group *group; + + cf = flb_cf_yaml_create(NULL, FLB_006, NULL, 0); + TEST_CHECK(cf != NULL); + if (!cf) { + exit(EXIT_FAILURE); + } + + /* Total number of sections */ + TEST_CHECK(mk_list_size(&cf->sections) == 4); + + /* Check number sections per list */ + TEST_CHECK(mk_list_size(&cf->upstream_servers) == 2); + TEST_CHECK(mk_list_size(&cf->parsers) == 0); + TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 0); + TEST_CHECK(mk_list_size(&cf->customs) == 0); + TEST_CHECK(mk_list_size(&cf->inputs) == 1); + TEST_CHECK(mk_list_size(&cf->filters) == 0); + TEST_CHECK(mk_list_size(&cf->outputs) == 1); + TEST_CHECK(mk_list_size(&cf->others) == 0); + + /* check upstream servers */ + idx = 0; + mk_list_foreach(head, &cf->upstream_servers) { + s = mk_list_entry(head, struct flb_cf_section, _head_section); + + switch (idx) { + case 0: + v = flb_cf_section_property_get(cf, s, "name"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "forward-balancing") == 0); + + /* iterate node/groups */ + TEST_CHECK(mk_list_size(&s->groups) == 3); + + g_idx = 0; + mk_list_foreach(g_head, &s->groups) { + group = mk_list_entry(g_head, struct flb_cf_group, _head); + TEST_CHECK(group != NULL); + TEST_CHECK(strcmp(group->name, "upstream_node") == 0); + + switch (g_idx) { + case 0: + v = cfl_kvlist_fetch(group->properties, "name"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "node-1") == 0); + + v = cfl_kvlist_fetch(group->properties, "host"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "127.0.0.1") == 0); + + v = cfl_kvlist_fetch(group->properties, "port"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "43000") == 0); + break; + + case 1: + v = cfl_kvlist_fetch(group->properties, "name"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "node-2") == 0); + + v = cfl_kvlist_fetch(group->properties, "host"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "127.0.0.1") == 0); + + v = cfl_kvlist_fetch(group->properties, "port"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "44000") == 0); + break; + case 2: + v = cfl_kvlist_fetch(group->properties, "name"); + TEST_CHECK(v != NULL); + + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "node-3") == 0); + break; + }; + g_idx++; + } + break; + case 1: + v = flb_cf_section_property_get(cf, s, "name"); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "forward-balancing-2") == 0); + + g_idx = 0; + mk_list_foreach(g_head, &s->groups) { + group = mk_list_entry(g_head, struct flb_cf_group, _head); + TEST_CHECK(group != NULL); + TEST_CHECK(strcmp(group->name, "upstream_node") == 0); + + switch (g_idx) { + case 0: + v = cfl_kvlist_fetch(group->properties, "name"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "node-A") == 0); + + v = cfl_kvlist_fetch(group->properties, "host"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "192.168.1.10") == 0); + + v = cfl_kvlist_fetch(group->properties, "port"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "50000") == 0); + + break; + case 1: + v = cfl_kvlist_fetch(group->properties, "name"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "node-B") == 0); + + v = cfl_kvlist_fetch(group->properties, "host"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "192.168.1.11") == 0); + + v = cfl_kvlist_fetch(group->properties, "port"); + TEST_CHECK(v != NULL); + TEST_CHECK(v->type == CFL_VARIANT_STRING); + TEST_CHECK(strcmp(v->data.as_string, "51000") == 0); + break; + }; + g_idx++; + } + + break; + }; + idx++; + } + + flb_cf_destroy(cf); +} + TEST_LIST = { { "basic" , test_basic}, { "customs section", test_customs_section}, @@ -464,5 +842,9 @@ TEST_LIST = { { "parsers file conf", test_parser_conf}, { "camel_case_key", test_camel_case_key}, { "processors", test_processors}, + { "parsers_and_multiline_parsers", test_parsers_and_multiline_parsers}, + { "stream_processor", test_stream_processor}, + { "plugins", test_plugins}, + { "upstream_servers", test_upstream_servers}, { 0 } }; diff --git a/tests/internal/data/config_format/yaml/extra_parser.yaml b/tests/internal/data/config_format/yaml/extra_parser.yaml new file mode 100644 index 00000000000..abbf29bd98c --- /dev/null +++ b/tests/internal/data/config_format/yaml/extra_parser.yaml @@ -0,0 +1,16 @@ +parsers: + - name: json-2 + format: json + +multiline_parsers: + - name: exception_test-2 + type: regex + flush_timeout: 1000 + rules: + - state: start_state + regex: "/(Dec \\d+ \\d+\\:\\d+\\:\\d+)(.*)/" + next_state: cont + + - state: cont + regex: "/^\\s+at.*/" + next_state: cont diff --git a/tests/internal/data/config_format/yaml/parsers_and_multiline_parsers.yaml b/tests/internal/data/config_format/yaml/parsers_and_multiline_parsers.yaml new file mode 100644 index 00000000000..637c93ea3b0 --- /dev/null +++ b/tests/internal/data/config_format/yaml/parsers_and_multiline_parsers.yaml @@ -0,0 +1,42 @@ +--- +service: + log_level: info + #parsers_file: parsers_multiline.conf + +includes: + - extra_parser.yaml + +parsers: + - name: json + format: json + + - name: docker + format: json + time_key: time + time_format: "%Y-%m-%dT%H:%M:%S.%L" + time_keep: true + +multiline_parsers: + - name: exception_test + type: regex + flush_timeout: 1000 + rules: + - state: start_state + regex: "/(Dec \\d+ \\d+\\:\\d+\\:\\d+)(.*)/" + next_state: cont + + - state: cont + regex: "/^\\s+at.*/" + next_state: cont + +pipeline: + inputs: + - name: tail + path: ../test_multiline.log + read_from_head: true + multiline.parser: multiline-regex-test + + outputs: + - name: stdout + match: '*' + format: json_lines diff --git a/tests/internal/data/config_format/yaml/plugins.yaml b/tests/internal/data/config_format/yaml/plugins.yaml new file mode 100644 index 00000000000..79378f6e55b --- /dev/null +++ b/tests/internal/data/config_format/yaml/plugins.yaml @@ -0,0 +1,16 @@ +--- + +plugins: + - /path/to/out_gstdout.so + - /path/to/out_fluent.so + +service: + log_level: info + +pipeline: + inputs: + - name: random + + outputs: + - name: gstdout + match: '*' diff --git a/tests/internal/data/config_format/yaml/stream_processor.yaml b/tests/internal/data/config_format/yaml/stream_processor.yaml new file mode 100644 index 00000000000..f098cc6b027 --- /dev/null +++ b/tests/internal/data/config_format/yaml/stream_processor.yaml @@ -0,0 +1,22 @@ +--- + +stream_processor: + - name: create_results + exec: CREATE STREAM results WITH (tag='500_error') AS SELECT * FROM STREAM:tail.0 WHERE http_status=500; + + - name: select_results + exec: SELECT * FROM STREAM:results WHERE http_status=500; + +service: + log_level: info + +pipeline: + inputs: + - name: tail + path: test_multiline.log + read_from_head: true + + outputs: + - name: stdout + match: '*' + format: json_lines diff --git a/tests/internal/data/config_format/yaml/upstream.yaml b/tests/internal/data/config_format/yaml/upstream.yaml new file mode 100644 index 00000000000..9dcf0ffb499 --- /dev/null +++ b/tests/internal/data/config_format/yaml/upstream.yaml @@ -0,0 +1,38 @@ +--- + +upstream_servers: + - name: forward-balancing + nodes: + - name: node-1 + host: 127.0.0.1 + port: 43000 + + - name: node-2 + host: 127.0.0.1 + port: 44000 + + - name: node-3 + host: 127.0.0.1 + port: 45000 + tls: true + tls_verify: false + shared_key: secret + + - name: forward-balancing-2 + nodes: + - name: node-A + host: 192.168.1.10 + port: 50000 + + - name: node-B + host: 192.168.1.11 + port: 51000 + +pipeline: + inputs: + - name: random + + outputs: + - name: stdout + match: '*' + diff --git a/tests/runtime/processor_content_modifier.c b/tests/runtime/processor_content_modifier.c index ac9cc5eac31..739c640978a 100644 --- a/tests/runtime/processor_content_modifier.c +++ b/tests/runtime/processor_content_modifier.c @@ -2,8 +2,7 @@ /* Fluent Bit * ========== - * Copyright (C) 2019-2024 The Fluent Bit Authors - * Copyright (C) 2015-2018 Treasure Data Inc. + * Copyright (C) 2015-2024 The Fluent Bit Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.