Merge branch 'fluent:master' into timestamp-encode-decode-big-endian

CMakeLists.txt

@@ -8,7 +8,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  3)
 set(FLB_VERSION_MINOR  1)
-set(FLB_VERSION_PATCH  6)
+set(FLB_VERSION_PATCH  7)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -165,6 +165,22 @@ option(FLB_CONFIG_YAML         "Enable YAML config format"    Yes)
 # List of plugins available and defaults for each option
 include(cmake/plugins_options.cmake)
 
+
+# Event loop backend (advanced users only)
+# ----------------------------------------
+# The following options are for advanced users only, they are used to enable
+# specific event loop backends. By default, the build system will try to
+# detect the best backend for the current Operating System. If you want to
+# force a specific backend, you can use ONE of the following options.
+#
+# Note that these options will enable the proper option for Monkey library.
+option(FLB_EVENT_LOOP_EPOLL     "Enable epoll(2) event loop backend"  No)
+option(FLB_EVENT_LOOP_POLL      "Enable poll(2) event loop backend"   No)
+option(FLB_EVENT_LOOP_KQUEUE    "Enable kqueue(2) event loop backend" No)
+option(FLB_EVENT_LOOP_SELECT    "Enable select(2) event loop backend" No)
+option(FLB_EVENT_LOOP_LIBEVENT  "Enable libevent event loop backend"  No)
+
+
 if(DEFINED FLB_NIGHTLY_BUILD AND NOT "${FLB_NIGHTLY_BUILD}" STREQUAL "")
   FLB_DEFINITION_VAL(FLB_NIGHTLY_BUILD ${FLB_NIGHTLY_BUILD})
 endif()
@@ -501,12 +517,38 @@ if(FLB_TESTS_RUNTIME)
   FLB_OPTION(FLB_HTTP_SERVER  ON)
 endif()
 
-# MK Core
+# Monkey Core Library
 macro(MK_SET_OPTION option value)
   set(${option} ${value} CACHE INTERNAL "" FORCE)
 endmacro()
-MK_SET_OPTION(MK_SYSTEM_MALLOC  ON)
-MK_SET_OPTION(MK_DEBUG          ON)
+MK_SET_OPTION(MK_SYSTEM_MALLOC    ON)
+MK_SET_OPTION(MK_DEBUG            ON)
+
+# Monkey backend event loop
+if (FLB_EVENT_LOOP_EPOLL)
+  MK_SET_OPTION(MK_EVENT_LOOP_EPOLL ON)
+  FLB_DEFINITION(FLB_EVENT_LOOP_EPOLL)
+  message(STATUS "FLB Event loop backend > epoll(2)")
+elseif (FLB_EVENT_LOOP_POLL)
+  MK_SET_OPTION(MK_EVENT_LOOP_POLL ON)
+  FLB_DEFINITION(FLB_EVENT_LOOP_POLL)
+  message(STATUS "FLB Event loop backend > poll(2)")
+elseif (FLB_EVENT_LOOP_KQUEUE)
+  MK_SET_OPTION(MK_EVENT_LOOP_KQUEUE ON)
+  FLB_DEFINITION(FLB_EVENT_LOOP_KQUEUE)
+  message(STATUS "FLB Event loop backend > kqueue(2)")
+elseif (FLB_EVENT_LOOP_SELECT)
+  MK_SET_OPTION(MK_EVENT_LOOP_SELECT ON)
+  FLB_DEFINITION(FLB_EVENT_LOOP_SELECT)
+  message(STATUS "FLB Event loop backend > select(2)")
+elseif (FLB_EVENT_LOOP_LIBEVENT)
+  MK_SET_OPTION(MK_EVENT_LOOP_LIBEVENT ON)
+  FLB_DEFINITION(FLB_EVENT_LOOP_LIBEVENT)
+  message(STATUS "FLB Event loop backend > libevent")
+else()
+  FLB_DEFINITION(FLB_EVENT_LOOP_AUTO_DISCOVERY)
+  message(STATUS "FLB Event loop backend > auto discovery (Monkey library)")
+endif()
 
 # Build Monkey HTTP Server
 if(FLB_HTTP_SERVER)

dockerfiles/Dockerfile

@@ -11,7 +11,7 @@
 # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/
 
 # Set this to the current release version: it gets done so as part of the release.
-ARG RELEASE_VERSION=3.1.6
+ARG RELEASE_VERSION=3.1.7
 
 # For multi-arch builds - assumption is running on an AMD64 host
 FROM multiarch/qemu-user-static:x86_64-arm as qemu-arm32

fluent-bit-3.1.6.bb → fluent-bit-3.1.7.bb

@@ -16,7 +16,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93"
 SECTION = "net"
 
 PR = "r0"
-PV = "3.1.6"
+PV = "3.1.7"
 
 SRCREV = "v${PV}"
 SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1"

include/fluent-bit/flb_metrics.h

@@ -39,6 +39,7 @@
 #include <cmetrics/cmt_encode_msgpack.h>
 #include <cmetrics/cmt_encode_splunk_hec.h>
 #include <cmetrics/cmt_encode_cloudwatch_emf.h>
+#include <cmetrics/cmt_decode_statsd.h>
 #include <cmetrics/cmt_filter.h>
 
 /* Metrics IDs for general purpose (used by core and Plugins */

include/fluent-bit/flb_network.h

@@ -40,7 +40,7 @@
 /* Network connection setup */
 struct flb_net_setup {
     /* enable/disable keepalive support */
-    char keepalive;
+    int keepalive;
 
     /* max time in seconds that a keepalive connection can be idle */
     int keepalive_idle_timeout;

include/fluent-bit/flb_parser.h

@@ -83,6 +83,7 @@ static inline time_t flb_parser_tm2time(const struct flb_tm *src,
 
     tmp = src->tm;
     if (use_system_timezone) {
+        tmp.tm_isdst = -1;
         res = mktime(&tmp);
     } else {
         res = timegm(&tmp) - flb_tm_gmtoff(src);

include/fluent-bit/flb_upstream_node.h

@@ -36,6 +36,7 @@ struct flb_upstream_node {
 #ifdef FLB_HAVE_TLS
     /* TLS: given configuration */
     int tls_verify;           /* Verify certs (default: true) */
+    int tls_verify_hostname;  /* Verify hostname (default: false) */
     int tls_debug;            /* mbedtls debug level          */
     char *tls_ca_path;        /* Path to certificates         */
     char *tls_ca_file;        /* CA root cert                 */
@@ -65,6 +66,7 @@ struct flb_upstream_node {
 struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t host,
                                                    flb_sds_t port,
                                                    int tls, int tls_verify,
+                                                   int tls_verify_hostname,
                                                    int tls_debug,
                                                    const char *tls_vhost,
                                                    const char *tls_ca_path,

lib/cmetrics/CMakeLists.txt

@@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 # CMetrics Version
 set(CMT_VERSION_MAJOR  0)
 set(CMT_VERSION_MINOR  9)
-set(CMT_VERSION_PATCH  3)
+set(CMT_VERSION_PATCH  4)
 set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}")
 
 # Include helpers

lib/cmetrics/include/cmetrics/cmt_decode_statsd.h

@@ -0,0 +1,60 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  CMetrics
+ *  ========
+ *  Copyright 2021-2022 The CMetrics Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+#ifndef CMT_DECODE_STATSD_H
+#define CMT_DECODE_STATSD_H
+
+#include <cmetrics/cmetrics.h>
+
+#define CMT_DECODE_STATSD_TYPE_COUNTER 1
+#define CMT_DECODE_STATSD_TYPE_GAUGE   2
+#define CMT_DECODE_STATSD_TYPE_TIMER   3
+#define CMT_DECODE_STATSD_TYPE_SET     4
+
+#define CMT_DECODE_STATSD_SUCCESS                  0
+#define CMT_DECODE_STATSD_ALLOCATION_ERROR         1
+#define CMT_DECODE_STATSD_UNEXPECTED_ERROR         2
+#define CMT_DECODE_STATSD_INVALID_ARGUMENT_ERROR   3
+#define CMT_DECODE_STATSD_UNEXPECTED_METRIC_TYPE   4
+#define CMT_DECODE_STATSD_DECODE_ERROR             5
+#define CMT_DECODE_STATSD_UNPACK_ERROR             6
+#define CMT_DECODE_STATSD_UNSUPPORTED_METRIC_TYPE  7
+#define CMT_DECODE_STATSD_INVALID_TAG_FORMAT_ERROR 8
+
+#define CMT_DECODE_STATSD_GAUGE_OBSERVER     1 << 0
+
+/*
+ * The "cmt_statsd_message" represents a single line in UDP packet.
+ * It's just a bunch of pointers to ephemeral buffer.
+ */
+struct cmt_statsd_message {
+    char *bucket;
+    int bucket_len;
+    char *value;
+    char *labels;
+    int value_len;
+    int type;
+    double sample_rate;
+};
+
+int cmt_decode_statsd_create(struct cmt **out_cmt, char *in_buf, size_t in_size, int flags);
+void cmt_decode_statsd_destroy(struct cmt *cmt);
+
+#endif

lib/cmetrics/src/CMakeLists.txt

@@ -35,6 +35,7 @@ set(src
   cmt_encode_influx.c
   cmt_encode_msgpack.c
   cmt_decode_msgpack.c
+  cmt_decode_statsd.c
   cmt_mpack_utils.c
   external/remote.pb-c.c
   external/types.pb-c.c

lib/cmetrics/src/cmt_cat.c

@@ -346,20 +346,16 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped,
 int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram,
                       struct cmt_map *filtered_map)
 {
-    int i;
-    double val;
     int ret;
     char **labels = NULL;
     struct cmt_map *map;
     struct cmt_opts *opts;
     struct cmt_histogram *hist;
-    uint64_t timestamp;
     struct cmt_histogram_buckets *buckets;
     int64_t buckets_count;
 
     map = histogram->map;
     opts = map->opts;
-    timestamp = cmt_metric_get_timestamp(&map->metric);
 
     ret = cmt_cat_copy_label_keys(map, (char **) &labels);
     if (ret == -1) {
@@ -433,6 +429,8 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary,
                              quantiles,
                              map->label_count, labels);
     if (!sum) {
+        free(labels);
+        free(quantiles);
         return -1;
     }
 

lib/cmetrics/src/cmt_counter.c

@@ -107,8 +107,8 @@ int cmt_counter_inc(struct cmt_counter *counter,
                                 counter->map, labels_count, label_vals,
                                 CMT_TRUE);
     if (!metric) {
-        cmt_log_error(counter->cmt, "unable to retrieve metric: %s for counter %s_%s_%s",
-                      counter->map, counter->opts.ns, counter->opts.subsystem,
+        cmt_log_error(counter->cmt, "unable to retrieve metric for counter %s_%s_%s",
+                      counter->opts.ns, counter->opts.subsystem,
                       counter->opts.name);
         return -1;
     }
@@ -125,8 +125,8 @@ int cmt_counter_add(struct cmt_counter *counter, uint64_t timestamp, double val,
                                 counter->map, labels_count, label_vals,
                                 CMT_TRUE);
     if (!metric) {
-        cmt_log_error(counter->cmt, "unable to retrieve metric: %s for counter %s_%s_%s",
-                      counter->map, counter->opts.ns, counter->opts.subsystem,
+        cmt_log_error(counter->cmt, "unable to retrieve metric for counter %s_%s_%s",
+                      counter->opts.ns, counter->opts.subsystem,
                       counter->opts.name);
         return -1;
     }
@@ -144,8 +144,8 @@ int cmt_counter_set(struct cmt_counter *counter, uint64_t timestamp, double val,
                                 labels_count, label_vals,
                                 CMT_TRUE);
     if (!metric) {
-        cmt_log_error(counter->cmt, "unable to retrieve metric: %s for counter %s_%s_%s",
-                      counter->map, counter->opts.ns, counter->opts.subsystem,
+        cmt_log_error(counter->cmt, "unable to retrieve metric for counter %s_%s_%s",
+                      counter->opts.ns, counter->opts.subsystem,
                       counter->opts.name);
         return -1;
     }
@@ -170,8 +170,8 @@ int cmt_counter_get_val(struct cmt_counter *counter,
                                  counter->map, labels_count, label_vals,
                                  &val);
     if (ret == -1) {
-        cmt_log_error(counter->cmt, "unable to retrieve metric: %s for counter %s_%s_%s",
-                      counter->map, counter->opts.ns, counter->opts.subsystem,
+        cmt_log_error(counter->cmt, "unable to retrieve metric for counter %s_%s_%s",
+                      counter->opts.ns, counter->opts.subsystem,
                       counter->opts.name);
         return -1;
     }

lib/cmetrics/src/cmt_decode_prometheus_remote_write.c

@@ -663,6 +663,7 @@ int cmt_decode_prometheus_remote_write_create(struct cmt **out_cmt, char *in_buf
                                               (uint8_t *) in_buf);
     if (write == NULL) {
         result = CMT_DECODE_PROMETHEUS_REMOTE_WRITE_UNPACK_ERROR;
+        cmt_destroy(cmt);
         return result;
     }