zstd: introduce zstd compression support in Fluent Bit

.github/workflows/call-build-macos.yaml

@@ -76,6 +76,8 @@ jobs:
         config:
           - name: "Apple Silicon macOS runner"
             runner: macos-14
+          - name: "Intel macOS runner"
+            runner: macos-14-large
 
     steps:
       - name: Checkout repository

CMakeLists.txt

@@ -485,6 +485,15 @@ add_subdirectory(${FLB_PATH_LIB_MPACK} EXCLUDE_FROM_ALL)
 # Miniz (zip)
 add_subdirectory(${FLB_PATH_LIB_MINIZ} EXCLUDE_FROM_ALL)
 
+# Zstd (zstd)
+set(ZSTD_BUILD_STATIC ON)
+set(ZSTD_BUILD_SHARED OFF)
+set(ZSTD_BUILD_COMPRESSION ON)
+set(ZSTD_BUILD_DECOMPRESSION ON)
+set(ZSTD_BUILD_DICTBUILDER OFF)
+set(ZSTD_BUILD_DEPRECATED OFF)
+add_subdirectory(${FLB_PATH_LIB_ZSTD}/build/cmake EXCLUDE_FROM_ALL)
+
 # ring buffer library
 add_subdirectory(${FLB_PATH_LIB_RING_BUFFER} EXCLUDE_FROM_ALL)
 

cmake/headers.cmake

@@ -34,6 +34,7 @@ include_directories(
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CTRACES}/include
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CPROFILES}/include
   ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RING_BUFFER}/lwrb/src/include
+  ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_ZSTD}/lib
 
   ${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include
   ${CMAKE_CURRENT_BINARY_DIR}/lib/cmetrics

cmake/libraries.cmake

@@ -25,3 +25,4 @@ set(FLB_PATH_LIB_SNAPPY      "lib/snappy-fef67ac")
 set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.4.0")
 set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb")
 set(FLB_PATH_LIB_WASM_MICRO_RUNTIME  "lib/wasm-micro-runtime-WAMR-1.3.3")
+set(FLB_PATH_LIB_ZSTD        "lib/zstd-1.5.6")

include/fluent-bit/flb_hash_table.h

@@ -57,6 +57,7 @@ struct flb_hash_table {
     int max_entries;
     int total_count;
     int cache_ttl;
+    int case_sensitivity;
     size_t size;
     struct mk_list entries;
     struct flb_hash_table_chain *table;
@@ -67,6 +68,8 @@ struct flb_hash_table *flb_hash_table_create_with_ttl(int cache_ttl, int evict_m
                                                       size_t size, int max_entries);
 void flb_hash_table_destroy(struct flb_hash_table *ht);
 
+void flb_hash_table_set_case_sensitivity(struct flb_hash_table *ht, int status);
+
 int flb_hash_table_add(struct flb_hash_table *ht,
                        const char *key, int key_len,
                        void *val, ssize_t val_size);

include/fluent-bit/flb_http_client.h

@@ -371,6 +371,8 @@ int flb_http_bearer_auth(struct flb_http_client *c,
                         const char *token);
 int flb_http_set_keepalive(struct flb_http_client *c);
 int flb_http_set_content_encoding_gzip(struct flb_http_client *c);
+int flb_http_set_content_encoding_zstd(struct flb_http_client *c);
+
 int flb_http_set_callback_context(struct flb_http_client *c,
                                   struct flb_callback *cb_ctx);
 int flb_http_set_response_test(struct flb_http_client *c, char *test_name,

include/fluent-bit/flb_zstd.h

@@ -0,0 +1,29 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2024 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_ZSTD_H
+#define FLB_ZSTD_H
+
+#include <fluent-bit/flb_info.h>
+#include <zstd.h>
+
+size_t flb_zstd_compress(void *in_data, size_t in_len, void **out_data, size_t *out_len);
+size_t flb_zstd_uncompress(void *in_data, size_t in_len, void **out_data, size_t *out_len);
+
+#endif

lib/monkey/CMakeLists.txt

@@ -23,7 +23,7 @@ endif()
 # Monkey Version
 set(MK_VERSION_MAJOR  1)
 set(MK_VERSION_MINOR  8)
-set(MK_VERSION_PATCH  1)
+set(MK_VERSION_PATCH  2)
 set(MK_VERSION_STR "${MK_VERSION_MAJOR}.${MK_VERSION_MINOR}.${MK_VERSION_PATCH}")
 
 # Output paths

lib/monkey/deps/mbedtls-2.4.2/.gitignore

@@ -0,0 +1,25 @@
+CMakeCache.txt
+CMakeFiles
+CTestTestfile.cmake
+cmake_install.cmake
+Testing
+Coverage
+*.gcno
+*.gcda
+
+# generated by scripts/memory.sh
+massif-*
+
+# MSVC files generated by CMake:
+/*.sln
+/*.vcxproj
+/*.filters
+
+# MSVC build artifacts:
+*.exe
+*.pdb
+*.ilk
+*.lib
+
+# CMake generates *.dir/ folders for in-tree builds (used by MSVC projects), ignore all of those:
+*.dir/

lib/monkey/deps/mbedtls-2.4.2/.travis.yml

@@ -0,0 +1,39 @@
+language: c
+compiler:
+- clang
+- gcc
+sudo: false
+cache: ccache
+script:
+- tests/scripts/recursion.pl library/*.c
+- tests/scripts/check-generated-files.sh
+- tests/scripts/check-doxy-blocks.pl
+- tests/scripts/check-names.sh
+- tests/scripts/doxygen.sh
+- cmake -D CMAKE_BUILD_TYPE:String="Check" .
+- make
+- make test
+- programs/test/selftest
+- OSSL_NO_DTLS=1 tests/compat.sh
+- tests/ssl-opt.sh -e '\(DTLS\|SCSV\).*openssl'
+- tests/scripts/test-ref-configs.pl
+- tests/scripts/curves.pl
+- tests/scripts/key-exchanges.pl
+after_failure:
+- tests/scripts/travis-log-failure.sh
+env:
+  global:
+    secure: "barHldniAfXyoWOD/vcO+E6/Xm4fmcaUoC9BeKW+LwsHqlDMLvugaJnmLXkSpkbYhVL61Hzf3bo0KPJn88AFc5Rkf8oYHPjH4adMnVXkf3B9ghHCgznqHsAH3choo6tnPxaFgOwOYmLGb382nQxfE5lUdvnM/W/psQjWt66A1+k="
+
+addons:
+  apt:
+    packages:
+    - doxygen
+    - graphviz
+  coverity_scan:
+    project:
+      name: "ARMmbed/mbedtls"
+    notification_email: [email protected]
+    build_command_prepend:
+    build_command: make
+    branch_pattern: coverity_scan

lib/monkey/deps/mbedtls-2.4.2/CMakeLists.txt

@@ -0,0 +1,166 @@
+cmake_minimum_required(VERSION 2.6)
+project("mbed TLS" C)
+
+option(USE_PKCS11_HELPER_LIBRARY "Build mbed TLS with the pkcs11-helper library." OFF)
+option(ENABLE_ZLIB_SUPPORT "Build mbed TLS with zlib library." OFF)
+
+option(ENABLE_PROGRAMS "Build mbed TLS programs." ON)
+
+option(UNSAFE_BUILD "Allow unsafe builds. These builds ARE NOT SECURE." OFF)
+
+# the test suites currently have compile errors with MSVC
+if(MSVC)
+    option(ENABLE_TESTING "Build mbed TLS tests." OFF)
+else()
+    option(ENABLE_TESTING "Build mbed TLS tests." ON)
+endif()
+
+# Warning string - created as a list for compatibility with CMake 2.8
+set(WARNING_BORDER "*******************************************************\n")
+set(NULL_ENTROPY_WARN_L1 "****  WARNING!  MBEDTLS_TEST_NULL_ENTROPY defined!\n")
+set(NULL_ENTROPY_WARN_L2 "****  THIS BUILD HAS NO DEFINED ENTROPY SOURCES\n")
+set(NULL_ENTROPY_WARN_L3 "****  AND IS *NOT* SUITABLE FOR PRODUCTION USE\n")
+
+set(NULL_ENTROPY_WARNING "${WARNING_BORDER}"
+                         "${NULL_ENTROPY_WARN_L1}"
+                         "${NULL_ENTROPY_WARN_L2}"
+                         "${NULL_ENTROPY_WARN_L3}"
+                         "${WARNING_BORDER}")
+
+find_package(Perl)
+if(PERL_FOUND)
+
+    # If NULL Entropy is configured, display an appropriate warning
+    execute_process(COMMAND ${PERL_EXECUTABLE} ${CMAKE_SOURCE_DIR}/scripts/config.pl -f ${CMAKE_SOURCE_DIR}/include/mbedtls/config.h get MBEDTLS_TEST_NULL_ENTROPY
+                        RESULT_VARIABLE result)
+    if(${result} EQUAL 0)
+        message(WARNING ${NULL_ENTROPY_WARNING})
+
+        if(NOT UNSAFE_BUILD)
+            message(FATAL_ERROR "\
+\n\
+Warning! You have enabled MBEDTLS_TEST_NULL_ENTROPY. \
+This option is not safe for production use and negates all security \
+It is intended for development use only. \
+\n\
+To confirm you want to build with this option, re-run cmake with the \
+option: \n\
+  cmake -DUNSAFE_BUILD=ON ")
+
+            return()
+        endif()
+    endif()
+endif()
+
+set(CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE}
+    CACHE STRING "Choose the type of build: None Debug Release Coverage ASan ASanDbg MemSan MemSanDbg Check CheckFull"
+    FORCE)
+
+string(REGEX MATCH "Clang" CMAKE_COMPILER_IS_CLANG "${CMAKE_C_COMPILER_ID}")
+
+if(CMAKE_COMPILER_IS_GNUCC)
+    # some warnings we want are not available with old GCC versions
+    # note: starting with CMake 2.8 we could use CMAKE_C_COMPILER_VERSION
+    execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion
+                    OUTPUT_VARIABLE GCC_VERSION)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -W -Wdeclaration-after-statement -Wwrite-strings")
+    if (GCC_VERSION VERSION_GREATER 4.5 OR GCC_VERSION VERSION_EQUAL 4.5)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wlogical-op")
+    endif()
+    if (GCC_VERSION VERSION_GREATER 4.8 OR GCC_VERSION VERSION_EQUAL 4.8)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wshadow")
+    endif()
+    set(CMAKE_C_FLAGS_RELEASE     "-O2")
+    set(CMAKE_C_FLAGS_DEBUG       "-O0 -g3")
+    set(CMAKE_C_FLAGS_COVERAGE    "-O0 -g3 --coverage")
+    set(CMAKE_C_FLAGS_ASAN        "-Werror -fsanitize=address -fno-common -O3")
+    set(CMAKE_C_FLAGS_ASANDBG     "-Werror -fsanitize=address -fno-common -O1 -g3 -fno-omit-frame-pointer -fno-optimize-sibling-calls ")
+    set(CMAKE_C_FLAGS_CHECK       "-Werror -Os")
+    set(CMAKE_C_FLAGS_CHECKFULL   "${CMAKE_C_FLAGS_CHECK} -Wcast-qual")
+endif(CMAKE_COMPILER_IS_GNUCC)
+
+if(CMAKE_COMPILER_IS_CLANG)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -W -Wdeclaration-after-statement -Wwrite-strings -Wpointer-arith -Wimplicit-fallthrough -Wshadow")
+    set(CMAKE_C_FLAGS_RELEASE     "-O2")
+    set(CMAKE_C_FLAGS_DEBUG       "-O0 -g3")
+    set(CMAKE_C_FLAGS_COVERAGE    "-O0 -g3 --coverage")
+    set(CMAKE_C_FLAGS_ASAN        "-Werror -fsanitize=address -fno-common -fsanitize=undefined -fno-sanitize-recover -O3")
+    set(CMAKE_C_FLAGS_ASANDBG     "-Werror -fsanitize=address -fno-common -fsanitize=undefined -fno-sanitize-recover -O1 -g3 -fno-omit-frame-pointer -fno-optimize-sibling-calls ")
+    set(CMAKE_C_FLAGS_MEMSAN      "-Werror -fsanitize=memory -O3")
+    set(CMAKE_C_FLAGS_MEMSANDBG   "-Werror -fsanitize=memory -O1 -g3 -fno-omit-frame-pointer -fno-optimize-sibling-calls -fsanitize-memory-track-origins=2")
+    set(CMAKE_C_FLAGS_CHECK       "-Werror -Os")
+endif(CMAKE_COMPILER_IS_CLANG)
+
+if(MSVC)
+    # Strictest warnings, and treat as errors
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX")
+endif(MSVC)
+
+if(CMAKE_BUILD_TYPE STREQUAL "Coverage")
+    if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
+        set(CMAKE_SHARED_LINKER_FLAGS "--coverage")
+    endif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
+endif(CMAKE_BUILD_TYPE STREQUAL "Coverage")
+
+if(LIB_INSTALL_DIR)
+else()
+    set(LIB_INSTALL_DIR lib)
+endif()
+
+include_directories(include/)
+
+if(ENABLE_ZLIB_SUPPORT)
+    find_package(ZLIB)
+
+    if(ZLIB_FOUND)
+        include_directories(${ZLIB_INCLUDE_DIR})
+    endif(ZLIB_FOUND)
+endif(ENABLE_ZLIB_SUPPORT)
+
+add_subdirectory(library)
+add_subdirectory(include)
+
+if(ENABLE_PROGRAMS)
+    add_subdirectory(programs)
+endif()
+
+ADD_CUSTOM_TARGET(apidoc
+    COMMAND doxygen doxygen/mbedtls.doxyfile
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
+if(ENABLE_TESTING)
+    enable_testing()
+
+    add_subdirectory(tests)
+
+    # additional convenience targets for Unix only
+    if(UNIX)
+
+        ADD_CUSTOM_TARGET(covtest
+            COMMAND make test
+            COMMAND programs/test/selftest
+            COMMAND tests/compat.sh
+            COMMAND tests/ssl-opt.sh
+        )
+
+        ADD_CUSTOM_TARGET(lcov
+            COMMAND rm -rf Coverage
+            COMMAND lcov --capture --initial --directory library/CMakeFiles/mbedtls.dir -o files.info
+            COMMAND lcov --capture --directory library/CMakeFiles/mbedtls.dir -o tests.info
+            COMMAND lcov --add-tracefile files.info --add-tracefile tests.info -o all.info
+            COMMAND lcov --remove all.info -o final.info '*.h'
+            COMMAND gendesc tests/Descriptions.txt -o descriptions
+            COMMAND genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --no-branch-coverage -o Coverage final.info
+            COMMAND rm -f files.info tests.info all.info final.info descriptions
+        )
+
+        ADD_CUSTOM_TARGET(memcheck
+            COMMAND sed -i.bak s+/usr/bin/valgrind+`which valgrind`+ DartConfiguration.tcl
+            COMMAND ctest -O memcheck.log -D ExperimentalMemCheck
+            COMMAND tail -n1 memcheck.log | grep 'Memory checking results:' > /dev/null
+            COMMAND rm -f memcheck.log
+            COMMAND mv DartConfiguration.tcl.bak DartConfiguration.tcl
+        )
+    endif(UNIX)
+endif()