From 72a81884551a6817a6227d6a185a372dc9d4e45a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 31 Aug 2024 12:05:44 -0600 Subject: [PATCH] lib: cmetrics: upgrade to v0.9.5 Signed-off-by: Eduardo Silva --- lib/cmetrics/.github/workflows/packages.yaml | 57 +++++++++++++++++++ lib/cmetrics/CMakeLists.txt | 2 +- lib/cmetrics/src/cmt_decode_prometheus.c | 13 ++++- .../src/cmt_decode_prometheus_remote_write.c | 18 +++++- lib/cmetrics/src/cmt_decode_statsd.c | 43 ++++++++++---- lib/cmetrics/src/cmt_encode_influx.c | 34 +++++++---- lib/cmetrics/src/cmt_encode_opentelemetry.c | 8 +-- .../tests/data/issue_fluent_bit_9267.txt | 1 + lib/cmetrics/tests/encoding.c | 53 +++++++++++++++++ lib/cmetrics/tests/prometheus_parser.c | 31 +++++++++- 10 files changed, 227 insertions(+), 33 deletions(-) create mode 100644 lib/cmetrics/tests/data/issue_fluent_bit_9267.txt diff --git a/lib/cmetrics/.github/workflows/packages.yaml b/lib/cmetrics/.github/workflows/packages.yaml index 8272af809d3..f0ed2335d6b 100644 --- a/lib/cmetrics/.github/workflows/packages.yaml +++ b/lib/cmetrics/.github/workflows/packages.yaml @@ -69,11 +69,68 @@ jobs: path: | ./*.${{matrix.format}} + build-macos-packages-amd64: + name: build macOS intel packages + strategy: + fail-fast: true + matrix: + config: + - format: productbuild + arch: intel + ext: pkg + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Build the ${{matrix.config.format}} packages + run: | + cmake . -DCPACK_GENERATOR=${{ matrix.config.format }} + echo ${{ matrix.config.format }} | xargs -I{} cpack -G {} + + - name: Store the master package artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.config.format }}-${{matrix.config.arch}} + path: | + ./*-${{matrix.config.arch}}.${{matrix.config.ext}} + + build-macos-packages-arm64: + name: build macOS Apple Silicon packages + strategy: + fail-fast: true + matrix: + config: + - format: productbuild + arch: apple + ext: pkg + runs-on: macos-14 + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Build the ${{matrix.config.format}} packages + run: | + cmake . -DCPACK_GENERATOR=${{ matrix.config.format }} + echo ${{ matrix.config.format }} | xargs -I{} cpack -G {} + + - name: Store the master package artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.config.format }}-${{matrix.config.arch}} + path: | + ./*-${{matrix.config.arch}}.${{matrix.config.ext}} + release: name: Create release and upload packages needs: - build-distro-packages-amd64 - build-distro-packages-arm64 + - build-macos-packages-amd64 + - build-macos-packages-arm64 + runs-on: ubuntu-latest permissions: contents: write diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index f5221d7f524..1cdd0db35da 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 0) set(CMT_VERSION_MINOR 9) -set(CMT_VERSION_PATCH 4) +set(CMT_VERSION_PATCH 5) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers diff --git a/lib/cmetrics/src/cmt_decode_prometheus.c b/lib/cmetrics/src/cmt_decode_prometheus.c index 3a87c34ee74..134cb92a54c 100644 --- a/lib/cmetrics/src/cmt_decode_prometheus.c +++ b/lib/cmetrics/src/cmt_decode_prometheus.c @@ -57,9 +57,16 @@ static void reset_context(struct cmt_decode_prometheus_context *context, } if (context->metric.ns) { - if (strcmp(context->metric.ns, "")) { + if ((void *) context->metric.ns != (void *) "") { /* when namespace is empty, "name" contains a pointer to the - * allocated string */ + * allocated string. + * + * Note : When the metric name doesn't include the namespace + * ns is set to a constant empty string and we need to + * differentiate that case from the case where an empty + * namespace is provided. + */ + free(context->metric.ns); } else { @@ -516,7 +523,7 @@ static int add_metric_histogram(struct cmt_decode_prometheus_context *context) "failed to parse bucket"); goto end; } - if (parse_uint64(sample->value1, + if (parse_uint64(sample->value1, bucket_defaults + bucket_index)) { /* Count is supposed to be integer, but apparently * some tools can generate count in a floating format. diff --git a/lib/cmetrics/src/cmt_decode_prometheus_remote_write.c b/lib/cmetrics/src/cmt_decode_prometheus_remote_write.c index 4aa91e676d4..3adeccefc4c 100644 --- a/lib/cmetrics/src/cmt_decode_prometheus_remote_write.c +++ b/lib/cmetrics/src/cmt_decode_prometheus_remote_write.c @@ -413,6 +413,8 @@ static int decode_histogram_points(struct cmt *cmt, destroy_label_list(&metric->labels); free(metric); + + return CMT_DECODE_PROMETHEUS_REMOTE_WRITE_DECODE_ERROR; } else { cfl_list_add(&metric->_head, &map->metrics); @@ -438,7 +440,13 @@ static int decode_histogram_points(struct cmt *cmt, } } else { - free(metric); + if (static_metric_detected == CMT_FALSE) { + destroy_label_list(&metric->labels); + + cfl_list_del(&metric->_head); + + free(metric); + } return CMT_DECODE_PROMETHEUS_REMOTE_WRITE_DECODE_ERROR; } @@ -452,7 +460,13 @@ static int decode_histogram_points(struct cmt *cmt, metric->hist_count = hist->count_float; } else { - free(metric); + if (static_metric_detected == CMT_FALSE) { + destroy_label_list(&metric->labels); + + cfl_list_del(&metric->_head); + + free(metric); + } return CMT_DECODE_PROMETHEUS_REMOTE_WRITE_DECODE_ERROR; } diff --git a/lib/cmetrics/src/cmt_decode_statsd.c b/lib/cmetrics/src/cmt_decode_statsd.c index 168f7ecc338..43b2213135c 100644 --- a/lib/cmetrics/src/cmt_decode_statsd.c +++ b/lib/cmetrics/src/cmt_decode_statsd.c @@ -108,7 +108,7 @@ static int decode_labels(struct cmt *cmt, size_t label_index; int label_found; char *label_kv, *colon; - cfl_sds_t label_k, label_v, tmp; + cfl_sds_t label_k = NULL, label_v = NULL, tmp = NULL; int result; struct cfl_list *head = NULL; struct cfl_list *kvs = NULL; @@ -157,7 +157,14 @@ static int decode_labels(struct cmt *cmt, } label_k = cfl_sds_create_len(label_kv, colon - label_kv); if (label_k == NULL) { + for (label_index = 0 ; label_index < 128 ; label_index++) { + if (value_index_list[label_index] != NULL) { + cfl_sds_destroy(value_index_list[label_index]); + } + } + free(value_index_list); + if (kvs != NULL) { cfl_utils_split_free(kvs); } @@ -167,7 +174,15 @@ static int decode_labels(struct cmt *cmt, label_v = cfl_sds_create_len(colon + 1, strlen(label_kv) - strlen(label_k) - 1); if (label_v == NULL) { cfl_sds_destroy(label_k); + + for (label_index = 0 ; label_index < 128 ; label_index++) { + if (value_index_list[label_index] != NULL) { + cfl_sds_destroy(value_index_list[label_index]); + } + } + free(value_index_list); + if (kvs != NULL) { cfl_utils_split_free(kvs); } @@ -190,7 +205,15 @@ static int decode_labels(struct cmt *cmt, if (label_index > 127) { cfl_sds_destroy(label_k); cfl_sds_destroy(label_v); + + for (label_index = 0 ; label_index < 128 ; label_index++) { + if (value_index_list[label_index] != NULL) { + cfl_sds_destroy(value_index_list[label_index]); + } + } + free(value_index_list); + if (kvs != NULL) { cfl_utils_split_free(kvs); } @@ -203,10 +226,12 @@ static int decode_labels(struct cmt *cmt, } if (result == CMT_DECODE_STATSD_SUCCESS) { - value_index_list[label_index] = (void *) label_v; + value_index_list[label_index] = (void *) cfl_sds_create_len(label_v, + cfl_sds_len(label_v)); } cfl_sds_destroy(label_k); + cfl_sds_destroy(label_v); } } @@ -225,20 +250,18 @@ static int decode_labels(struct cmt *cmt, } } - for (map_label_index = 0 ; - result == CMT_DECODE_STATSD_SUCCESS && - map_label_index < map_label_count ; - map_label_index++) { - label_v = (cfl_sds_t) value_index_list[map_label_index]; - cfl_sds_destroy(label_v); + for (label_index = 0 ; label_index < 128 ; label_index++) { + if (value_index_list[label_index] != NULL) { + cfl_sds_destroy(value_index_list[label_index]); + } } + free(value_index_list); + if (kvs != NULL) { cfl_utils_split_free(kvs); } - free(value_index_list); - return result; } diff --git a/lib/cmetrics/src/cmt_encode_influx.c b/lib/cmetrics/src/cmt_encode_influx.c index 1dfe3797d43..747355d3176 100644 --- a/lib/cmetrics/src/cmt_encode_influx.c +++ b/lib/cmetrics/src/cmt_encode_influx.c @@ -226,8 +226,9 @@ static void format_metric(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, { int i; int n; - int count = 0; + int static_count = 0; int static_labels = 0; + int has_namespace = CMT_FALSE; struct cmt_map_label *label_k; struct cmt_map_label *label_v; struct cfl_list *head; @@ -241,19 +242,26 @@ static void format_metric(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, opts = map->opts; /* Measurement */ - cfl_sds_cat_safe(buf, opts->ns, cfl_sds_len(opts->ns)); - - if (cfl_sds_len(opts->subsystem) > 0) { - cfl_sds_cat_safe(buf, "_", 1); - cfl_sds_cat_safe(buf, opts->subsystem, cfl_sds_len(opts->subsystem)); + if (cfl_sds_len(opts->ns) > 0) { + cfl_sds_cat_safe(buf, opts->ns, cfl_sds_len(opts->ns)); + if (cfl_sds_len(opts->subsystem) > 0) { + cfl_sds_cat_safe(buf, "_", 1); + cfl_sds_cat_safe(buf, opts->subsystem, cfl_sds_len(opts->subsystem)); + } + has_namespace = CMT_TRUE; + } + else { + has_namespace = CMT_FALSE; } /* Static labels (tags) */ static_labels = cmt_labels_count(cmt->static_labels); if (static_labels > 0) { - cfl_sds_cat_safe(buf, ",", 1); + if (has_namespace == CMT_TRUE) { + cfl_sds_cat_safe(buf, ",", 1); + } cfl_list_foreach(head, &cmt->static_labels->list) { - count++; + static_count++; slabel = cfl_list_entry(head, struct cmt_label, _head); /* key */ @@ -265,7 +273,7 @@ static void format_metric(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, /* val */ append_string(buf, slabel->val); - if (count < static_labels) { + if (static_count < static_labels) { cfl_sds_cat_safe(buf, ",", 1); } } @@ -274,7 +282,9 @@ static void format_metric(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, /* Labels / Tags */ n = cfl_list_size(&metric->labels); if (n > 0) { - cfl_sds_cat_safe(buf, ",", 1); + if (static_labels > 0 || has_namespace == CMT_TRUE) { + cfl_sds_cat_safe(buf, ",", 1); + } label_k = cfl_list_entry_first(&map->label_keys, struct cmt_map_label, _head); @@ -297,7 +307,9 @@ static void format_metric(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, } } - cfl_sds_cat_safe(buf, " ", 1); + if (has_namespace == CMT_TRUE || static_labels > 0 || n > 0) { + cfl_sds_cat_safe(buf, " ", 1); + } append_metric_value(map, buf, metric); } diff --git a/lib/cmetrics/src/cmt_encode_opentelemetry.c b/lib/cmetrics/src/cmt_encode_opentelemetry.c index 8c934186330..8378bc8694d 100644 --- a/lib/cmetrics/src/cmt_encode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_encode_opentelemetry.c @@ -711,13 +711,13 @@ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_binary_to_ result->bytes_value.len = cfl_sds_len(value->data.as_bytes); result->bytes_value.data = calloc(result->bytes_value.len, sizeof(char)); - if (result->bytes_value.data == NULL) { + if (result->bytes_value.data) { + memcpy(result->bytes_value.data, value->data.as_bytes, result->bytes_value.len); + } + else { otlp_any_value_destroy(result); - result = NULL; } - - memcpy(result->bytes_value.data, value->data.as_bytes, result->bytes_value.len); } return result; diff --git a/lib/cmetrics/tests/data/issue_fluent_bit_9267.txt b/lib/cmetrics/tests/data/issue_fluent_bit_9267.txt new file mode 100644 index 00000000000..979183b032f --- /dev/null +++ b/lib/cmetrics/tests/data/issue_fluent_bit_9267.txt @@ -0,0 +1 @@ +_bu_nn 0 171798732 \ No newline at end of file diff --git a/lib/cmetrics/tests/encoding.c b/lib/cmetrics/tests/encoding.c index 9d4478e4016..b962a468bd3 100644 --- a/lib/cmetrics/tests/encoding.c +++ b/lib/cmetrics/tests/encoding.c @@ -803,6 +803,58 @@ void test_influx() cmt_destroy(cmt); } +void test_influx_without_namespaces() +{ + uint64_t ts; + cfl_sds_t text; + struct cmt *cmt; + struct cmt_counter *c1; + struct cmt_counter *c2; + + char *out1 = \ + "test=1 1435658235000000123\n" + "host=calyptia.com,app=cmetrics test=2 1435658235000000123\n" + "host=aaa,app=bbb nosubsystem=1 1435658235000000123\n"; + + char *out2 = \ + "dev=Calyptia,lang=C test=1 1435658235000000123\n" + "dev=Calyptia,lang=C,host=calyptia.com,app=cmetrics test=2 1435658235000000123\n" + "dev=Calyptia,lang=C,host=aaa,app=bbb nosubsystem=1 1435658235000000123\n"; + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + c1 = cmt_counter_create(cmt, "", "", "test", "Static labels test", + 2, (char *[]) {"host", "app"}); + + ts = 1435658235000000123; + cmt_counter_inc(c1, ts, 0, NULL); + cmt_counter_inc(c1, ts, 2, (char *[]) {"calyptia.com", "cmetrics"}); + cmt_counter_inc(c1, ts, 2, (char *[]) {"calyptia.com", "cmetrics"}); + + c2 = cmt_counter_create(cmt, "", "", "nosubsystem", "No subsystem", + 2, (char *[]) {"host", "app"}); + + cmt_counter_inc(c2, ts, 2, (char *[]) {"aaa", "bbb"}); + + /* Encode to prometheus (no static labels) */ + text = cmt_encode_influx_create(cmt); + printf("%s\n", text); + TEST_CHECK(strcmp(text, out1) == 0); + cmt_encode_influx_destroy(text); + + /* append static labels */ + cmt_label_add(cmt, "dev", "Calyptia"); + cmt_label_add(cmt, "lang", "C"); + + text = cmt_encode_influx_create(cmt); + printf("%s\n", text); + TEST_CHECK(strcmp(text, out2) == 0); + cmt_encode_influx_destroy(text); + + cmt_destroy(cmt); +} + void test_splunk_hec() { uint64_t ts; @@ -1125,6 +1177,7 @@ TEST_LIST = { {"prometheus", test_prometheus}, {"text", test_text}, {"influx", test_influx}, + {"influx_without_namespaces", test_influx_without_namespaces}, {"splunk_hec", test_splunk_hec}, {"splunk_hec_floating_point", test_splunk_hec_floating_point}, {"splunk_hec_histogram", test_splunk_hec_histogram}, diff --git a/lib/cmetrics/tests/prometheus_parser.c b/lib/cmetrics/tests/prometheus_parser.c index 8cdef466a72..5fa0c889886 100644 --- a/lib/cmetrics/tests/prometheus_parser.c +++ b/lib/cmetrics/tests/prometheus_parser.c @@ -261,7 +261,7 @@ void test_escape_sequences() } void test_metric_without_labels() -{ +{ cfl_sds_t result; const char expected[] = @@ -776,7 +776,7 @@ void test_issue_fluent_bit_5541() "http_request_duration_seconds_bucket{le=\"0.25\"} 2 0\n" "http_request_duration_seconds_bucket{le=\"0.5\"} 2 0\n" "http_request_duration_seconds_bucket{le=\"0.75\"} 2 0\n" - "http_request_duration_seconds_bucket{le=\"1.0\"} 2 0\n" + "http_request_duration_seconds_bucket{le=\"1.0\"} 2 0\n" "http_request_duration_seconds_bucket{le=\"2.5\"} 2 0\n" "http_request_duration_seconds_bucket{le=\"5.0\"} 2 0\n" "http_request_duration_seconds_bucket{le=\"7.5\"} 2 0\n" @@ -1665,6 +1665,32 @@ void test_issue_fluent_bit_6534() cmt_decode_prometheus_destroy(cmt); } +void test_issue_fluent_bit_9267() +{ + char errbuf[256]; + int status; + cfl_sds_t result = NULL; + struct cmt *cmt; + struct cmt_decode_prometheus_parse_opts opts; + memset(&opts, 0, sizeof(opts)); + opts.errbuf = errbuf; + opts.errbuf_size = sizeof(errbuf); + cfl_sds_t in_buf = read_file(CMT_TESTS_DATA_PATH "/issue_fluent_bit_9267.txt"); + size_t in_size = cfl_sds_len(in_buf); + + cmt = NULL; + status = cmt_decode_prometheus_create(&cmt, in_buf, in_size, &opts); + TEST_CHECK(status == 0); + if (status) { + fprintf(stderr, "PARSE ERROR:\n======\n%s\n======\n", errbuf); + } + + if (cmt != NULL) { + cmt_decode_prometheus_destroy(cmt); + } + cfl_sds_destroy(in_buf); +} + TEST_LIST = { {"header_help", test_header_help}, {"header_type", test_header_type}, @@ -1697,5 +1723,6 @@ TEST_LIST = { {"pr_168", test_pr_168}, {"histogram_different_label_count", test_histogram_different_label_count}, {"issue_fluent_bit_6534", test_issue_fluent_bit_6534}, + {"issue_fluent_bit_9267", test_issue_fluent_bit_9267}, { 0 } };