From 5eb6a1083a32902713a8ca9e81d33017b8e53790 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Mon, 25 Sep 2023 18:52:58 +0200 Subject: [PATCH 1/9] log_event_encoder: switched data size type to overcome msvc issue Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_log_event_encoder.h | 24 +++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/include/fluent-bit/flb_log_event_encoder.h b/include/fluent-bit/flb_log_event_encoder.h index e55cc154349..61095944098 100644 --- a/include/fluent-bit/flb_log_event_encoder.h +++ b/include/fluent-bit/flb_log_event_encoder.h @@ -75,36 +75,46 @@ #define FLB_LOG_EVENT_APPEND_UNTIL_TERMINATOR -1 +/* Note: all of the size_t casts have been replaced with + * size_t pointer casts because there is an issue + * with msvc where it doesn't honor the type promotion + * when a small constant integer is hard-coded. + * + * This should not be a problem because according to + * the standard a size_t should be as large as the + * native register size just like pointers. + */ + #define FLB_LOG_EVENT_VALUE_LIST_TERMINATOR() \ (int) FLB_LOG_EVENT_APPEND_TERMINATOR_VALUE_TYPE #define FLB_LOG_EVENT_STRING_LENGTH_VALUE(length) \ (int) FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE, \ - (size_t) length + (size_t *) length #define FLB_LOG_EVENT_STRING_BODY_VALUE(buffer, length) \ (int) FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE, \ (char *) buffer, \ - (size_t) length + (size_t *) length #define FLB_LOG_EVENT_BINARY_LENGTH_VALUE(length) \ (int) FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE, \ - (size_t) length + (size_t *) length #define FLB_LOG_EVENT_BINARY_BODY_VALUE(buffer, length) \ (int) FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE, \ (char *) buffer, \ - (size_t) length + (size_t *) length #define FLB_LOG_EVENT_EXT_LENGTH_VALUE(type_, length) \ (int) FLB_LOG_EVENT_EXT_LENGTH_VALUE_TYPE, \ (int) type_, \ - (size_t) length + (size_t *) length #define FLB_LOG_EVENT_EXT_BODY_VALUE(buffer, length) \ (int) FLB_LOG_EVENT_EXT_BODY_VALUE_TYPE, \ (char *) buffer, \ - (size_t) length + (size_t *) length #define FLB_LOG_EVENT_TIMESTAMP_VALUE(value) \ (int) FLB_LOG_EVENT_TIMESTAMP_VALUE_TYPE, \ @@ -176,7 +186,7 @@ #define FLB_LOG_EVENT_MSGPACK_RAW_VALUE(buffer, length) \ (int) FLB_LOG_EVENT_MSGPACK_RAW_VALUE_TYPE, \ (char *) buffer, \ - (size_t) length + (size_t *) length #define FLB_LOG_EVENT_STRING_VALUE(buffer, length) \ FLB_LOG_EVENT_STRING_LENGTH_VALUE(length), \ From 570f42c9825692d15d508f6b81e26c32ad78bf7a Mon Sep 17 00:00:00 2001 From: David Korczynski Date: Tue, 26 Sep 2023 05:57:33 +0100 Subject: [PATCH 2/9] tests: fuzzers: fix fstore div-by-zero Signed-off-by: David Korczynski --- tests/internal/fuzzers/fstore_fuzzer.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/internal/fuzzers/fstore_fuzzer.c b/tests/internal/fuzzers/fstore_fuzzer.c index dd7a6cf8b96..92ecf5d7a2d 100644 --- a/tests/internal/fuzzers/fstore_fuzzer.c +++ b/tests/internal/fuzzers/fstore_fuzzer.c @@ -45,6 +45,20 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) struct flb_fstore_stream *st; struct flb_fstore_file *fsf; + /* Set flb_malloc_mod to be fuzzer-data dependent */ + if (size < 4) { + return 0; + } + flb_malloc_p = 0; + flb_malloc_mod = *(int*)data; + data += 4; + size -= 4; + + /* Avoid division by zero for modulo operations */ + if (flb_malloc_mod == 0) { + flb_malloc_mod = 1; + } + cio_utils_recursive_delete(FSF_STORE_PATH); fs = flb_fstore_create(FSF_STORE_PATH, FLB_FSTORE_FS); st = flb_fstore_stream_create(fs, "abc"); From 1aa1104120631f5d1bc1d12f3f9bd6d117dcc394 Mon Sep 17 00:00:00 2001 From: Phillip Whelan Date: Mon, 25 Sep 2023 16:39:43 -0300 Subject: [PATCH 3/9] filter_log_to_metrics: process the 'add_label' property, which was renamed from 'label'. Signed-off-by: Phillip Whelan --- plugins/filter_log_to_metrics/log_to_metrics.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/filter_log_to_metrics/log_to_metrics.c b/plugins/filter_log_to_metrics/log_to_metrics.c index cb7f285d882..a61e4827f34 100644 --- a/plugins/filter_log_to_metrics/log_to_metrics.c +++ b/plugins/filter_log_to_metrics/log_to_metrics.c @@ -268,7 +268,7 @@ static int set_labels(struct log_to_metrics_ctx *ctx, snprintf(label_keys[counter], MAX_LABEL_LENGTH - 1, "%s", kv->val); counter++; } - else if (strcasecmp(kv->key, "label") == 0) { + else if (strcasecmp(kv->key, "add_label") == 0) { split = flb_utils_split(kv->val, ' ', 1); if (mk_list_size(split) != 2) { flb_plg_error(ctx->ins, "invalid label, expected name and key"); From 9a68eef6c843f0ad35cdd3333c4ee1aab19b320c Mon Sep 17 00:00:00 2001 From: Phillip Whelan Date: Mon, 25 Sep 2023 16:40:13 -0300 Subject: [PATCH 4/9] filter_log_to_metrics: fix label test to use 'add_label'. Signed-off-by: Phillip Whelan --- tests/runtime/filter_log_to_metrics.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runtime/filter_log_to_metrics.c b/tests/runtime/filter_log_to_metrics.c index b8c8dba1dc2..eb38af61ce4 100644 --- a/tests/runtime/filter_log_to_metrics.c +++ b/tests/runtime/filter_log_to_metrics.c @@ -669,7 +669,7 @@ void flb_test_log_to_metrics_label(void) "metric_name", "test", "metric_description", "Counts messages", "kubernetes_mode", "off", - "label", "pod_name $kubernetes['pod_name']", + "add_label", "pod_name $kubernetes['pod_name']", NULL); out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); From 6e4bbeb75e09e1c1b1f7de0aca8fae861379814c Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 26 Sep 2023 10:15:14 +0200 Subject: [PATCH 5/9] tests: internal: stream_processor: fixed memory leak Signed-off-by: Leonardo Alminana --- tests/internal/stream_processor.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/internal/stream_processor.c b/tests/internal/stream_processor.c index c9e59f50407..6e9b631713c 100644 --- a/tests/internal/stream_processor.c +++ b/tests/internal/stream_processor.c @@ -517,6 +517,13 @@ static void test_window() usleep(800000); } + if (out_buf.buffer != NULL) { + flb_free(out_buf.buffer); + + out_buf.buffer = NULL; + out_buf.size = 0; + } + flb_sp_fd_event_test(task->window.fd, task, &out_buf); flb_info("[sp test] id=%i, SQL => '%s'", check->id, check->exec); @@ -560,13 +567,26 @@ static void test_window() /* Hopping event */ if ((t + 1) % check->window_hop_sec == 0) { + if (out_buf.buffer != NULL) { + flb_free(out_buf.buffer); + + out_buf.buffer = NULL; + out_buf.size = 0; + } + flb_sp_fd_event_test(task->window.fd_hop, task, &out_buf); } /* Window event */ if ((t + 1) % check->window_size_sec == 0 || (t + 1 > check->window_size_sec && (t + 1 - check->window_size_sec) % check->window_hop_sec == 0)) { - flb_free(out_buf.buffer); + if (out_buf.buffer != NULL) { + flb_free(out_buf.buffer); + + out_buf.buffer = NULL; + out_buf.size = 0; + } + flb_sp_fd_event_test(task->window.fd, task, &out_buf); } flb_free(data_buf.buffer); From 61dbbd8d457b8330626ff3b94dfb377876dbbc91 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 26 Sep 2023 10:45:56 +0200 Subject: [PATCH 6/9] tests: internal: stream_processor: fixed memory leak Signed-off-by: Leonardo Alminana --- tests/internal/stream_processor.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/internal/stream_processor.c b/tests/internal/stream_processor.c index 6e9b631713c..44e983a0689 100644 --- a/tests/internal/stream_processor.c +++ b/tests/internal/stream_processor.c @@ -503,6 +503,13 @@ static void test_window() /* We ingest the buffer every second */ for (t = 0; t < check->window_size_sec; t++) { + if (out_buf.buffer != NULL) { + flb_free(out_buf.buffer); + + out_buf.buffer = NULL; + out_buf.size = 0; + } + ret = flb_sp_do_test(sp, task, "samples", strlen("samples"), &data_buf, &out_buf); From 8269d80bf9455d0e31ae892fffb64f802f35fa86 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Mon, 25 Sep 2023 17:28:08 +0200 Subject: [PATCH 7/9] tests: internal: sp: added environment initialization calls Signed-off-by: Leonardo Alminana --- tests/internal/stream_processor.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/internal/stream_processor.c b/tests/internal/stream_processor.c index 44e983a0689..9afda0bf545 100644 --- a/tests/internal/stream_processor.c +++ b/tests/internal/stream_processor.c @@ -18,6 +18,7 @@ * limitations under the License. */ +#include #include #include #include @@ -200,6 +201,8 @@ static void invalid_queries() struct flb_sp *sp; struct flb_sp_task *task; + flb_init_env(); + /* Total number of checks for invalid queries */ checks = sizeof(invalid_query_checks) / sizeof(char *); @@ -244,6 +247,8 @@ static void test_select_keys() WSADATA wsa_data; #endif + flb_init_env(); + config = flb_calloc(1, sizeof(struct flb_config)); if (!config) { flb_errno(); @@ -330,6 +335,8 @@ static void test_select_subkeys() WSADATA wsa_data; #endif + flb_init_env(); + config = flb_calloc(1, sizeof(struct flb_config)); if (!config) { flb_errno(); @@ -458,6 +465,8 @@ static void test_window() WSADATA wsa_data; #endif + flb_init_env(); + config = flb_calloc(1, sizeof(struct flb_config)); if (!config) { flb_errno(); @@ -639,6 +648,8 @@ static void test_snapshot() WSADATA wsa_data; #endif + flb_init_env(); + config = flb_calloc(1, sizeof(struct flb_config)); if (!config) { flb_errno(); @@ -800,6 +811,8 @@ static void test_conv_from_str_to_num() #endif out_buf.buffer = NULL; + flb_init_env(); + config = flb_config_init(); config->evl = mk_event_loop_create(256); From b273c03623cd6783231c1c211b79ca3d7b05f0b1 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 26 Sep 2023 21:07:19 +0200 Subject: [PATCH 8/9] log_event_encoder: switched data size type to overcome msvc issue Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_log_event_encoder.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/flb_log_event_encoder.h b/include/fluent-bit/flb_log_event_encoder.h index 61095944098..8bca0b832ea 100644 --- a/include/fluent-bit/flb_log_event_encoder.h +++ b/include/fluent-bit/flb_log_event_encoder.h @@ -153,7 +153,7 @@ #define FLB_LOG_EVENT_INT64_VALUE(value) \ (int) FLB_LOG_EVENT_INT64_VALUE_TYPE, \ - (int64_t) value + (int64_t *) value #define FLB_LOG_EVENT_UINT8_VALUE(value) \ (int) FLB_LOG_EVENT_UINT8_VALUE_TYPE, \ @@ -169,7 +169,7 @@ #define FLB_LOG_EVENT_UINT64_VALUE(value) \ (int) FLB_LOG_EVENT_UINT64_VALUE_TYPE, \ - (uint64_t) value + (uint64_t *) value #define FLB_LOG_EVENT_DOUBLE_VALUE(value) \ (int) FLB_LOG_EVENT_DOUBLE_VALUE_TYPE, \ From 8f8bf7e6a12a2ff5c6d5cb1ee809c74d3bd5f9f8 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 27 Sep 2023 16:17:21 +0200 Subject: [PATCH 9/9] log_event_encoder: fixed 64 bit integer corner case Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_log_event_encoder.h | 65 +++++++++++++++------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/include/fluent-bit/flb_log_event_encoder.h b/include/fluent-bit/flb_log_event_encoder.h index 8bca0b832ea..6979634ac4c 100644 --- a/include/fluent-bit/flb_log_event_encoder.h +++ b/include/fluent-bit/flb_log_event_encoder.h @@ -28,6 +28,43 @@ #include +/* Note: + * 64 bit Windows : + * All of the size_t casts have been replaced with + * size_t pointer casts because there is an issue + * with msvc where it doesn't honor the type promotion + * when a small constant integer is hard-coded. + * + * This should not be a problem because according to + * the standard a size_t should be as large as the + * native register size just like pointers. + * + * 32 bit Windows : + * 64 bit integers are still defined as their specific + * types because the compiler handles them properly. + * + * Additionally, even though it would be preferrable to be + * able to use the optimize pragma to selectively disable + * the problematic optimizations it doesn't seem to work + * as expected. + */ + +#ifdef FLB_SYSTEM_WINDOWS +#ifdef _WIN64 +typedef size_t * flb_log_event_encoder_size_t; +typedef size_t * flb_log_event_encoder_int64_t; +typedef size_t * flb_log_event_encoder_uint64_t; +#else +typedef size_t * flb_log_event_encoder_size_t; +typedef int64_t flb_log_event_encoder_int64_t; +typedef uint64_t flb_log_event_encoder_uint64_t; +#endif +#else +typedef size_t flb_log_event_encoder_size_t; +typedef int64_t flb_log_event_encoder_int64_t; +typedef uint64_t flb_log_event_encoder_uint64_t; +#endif + #define FLB_EVENT_ENCODER_SUCCESS 0 #define FLB_EVENT_ENCODER_ERROR_UNSPECIFIED -1 #define FLB_EVENT_ENCODER_ERROR_ALLOCATION_ERROR -2 @@ -75,46 +112,36 @@ #define FLB_LOG_EVENT_APPEND_UNTIL_TERMINATOR -1 -/* Note: all of the size_t casts have been replaced with - * size_t pointer casts because there is an issue - * with msvc where it doesn't honor the type promotion - * when a small constant integer is hard-coded. - * - * This should not be a problem because according to - * the standard a size_t should be as large as the - * native register size just like pointers. - */ - #define FLB_LOG_EVENT_VALUE_LIST_TERMINATOR() \ (int) FLB_LOG_EVENT_APPEND_TERMINATOR_VALUE_TYPE #define FLB_LOG_EVENT_STRING_LENGTH_VALUE(length) \ (int) FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE, \ - (size_t *) length + (flb_log_event_encoder_size_t) length #define FLB_LOG_EVENT_STRING_BODY_VALUE(buffer, length) \ (int) FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE, \ (char *) buffer, \ - (size_t *) length + (flb_log_event_encoder_size_t) length #define FLB_LOG_EVENT_BINARY_LENGTH_VALUE(length) \ (int) FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE, \ - (size_t *) length + (flb_log_event_encoder_size_t) length #define FLB_LOG_EVENT_BINARY_BODY_VALUE(buffer, length) \ (int) FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE, \ (char *) buffer, \ - (size_t *) length + (flb_log_event_encoder_size_t) length #define FLB_LOG_EVENT_EXT_LENGTH_VALUE(type_, length) \ (int) FLB_LOG_EVENT_EXT_LENGTH_VALUE_TYPE, \ (int) type_, \ - (size_t *) length + (flb_log_event_encoder_size_t) length #define FLB_LOG_EVENT_EXT_BODY_VALUE(buffer, length) \ (int) FLB_LOG_EVENT_EXT_BODY_VALUE_TYPE, \ (char *) buffer, \ - (size_t *) length + (flb_log_event_encoder_size_t) length #define FLB_LOG_EVENT_TIMESTAMP_VALUE(value) \ (int) FLB_LOG_EVENT_TIMESTAMP_VALUE_TYPE, \ @@ -153,7 +180,7 @@ #define FLB_LOG_EVENT_INT64_VALUE(value) \ (int) FLB_LOG_EVENT_INT64_VALUE_TYPE, \ - (int64_t *) value + (flb_log_event_encoder_int64_t) value #define FLB_LOG_EVENT_UINT8_VALUE(value) \ (int) FLB_LOG_EVENT_UINT8_VALUE_TYPE, \ @@ -169,7 +196,7 @@ #define FLB_LOG_EVENT_UINT64_VALUE(value) \ (int) FLB_LOG_EVENT_UINT64_VALUE_TYPE, \ - (uint64_t *) value + (flb_log_event_encoder_uint64_t) value #define FLB_LOG_EVENT_DOUBLE_VALUE(value) \ (int) FLB_LOG_EVENT_DOUBLE_VALUE_TYPE, \ @@ -186,7 +213,7 @@ #define FLB_LOG_EVENT_MSGPACK_RAW_VALUE(buffer, length) \ (int) FLB_LOG_EVENT_MSGPACK_RAW_VALUE_TYPE, \ (char *) buffer, \ - (size_t *) length + (flb_log_event_encoder_size_t) length #define FLB_LOG_EVENT_STRING_VALUE(buffer, length) \ FLB_LOG_EVENT_STRING_LENGTH_VALUE(length), \