gzip: Merge distinguishing concatenated gzip function into flb_gzip m…

…odule

Signed-off-by: Hiroshi Hatake <[email protected]>

include/fluent-bit/flb_gzip.h

@@ -38,5 +38,6 @@ int flb_gzip_decompressor_dispatch(struct flb_decompression_context *context,
                                    void *out_data, size_t *out_size);
 
 int flb_is_http_session_gzip_compressed(struct mk_http_session *session);
+size_t flb_gzip_count(const char *data, size_t len, size_t **out_borders, size_t border_count);
 
 #endif

plugins/in_forward/fw_prot.c

@@ -1495,11 +1495,16 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
                         size_t *gzip_borders = NULL;
                         const size_t original_len = len;
 
-                        gzip_payloads_count = flb_gzip_concatenated_count(data, len);
+                        gzip_payloads_count = flb_gzip_count(data, len, NULL, 0);
                         flb_plg_debug(ctx->ins, "concatenated gzip payload count is %zd",
                                      gzip_payloads_count);
                         if (gzip_payloads_count > 0) {
-                            if (flb_gzip_concatenated_borders(data, len, &gzip_borders, gzip_payloads_count) < 0) {
+                            gzip_borders = (size_t *)flb_calloc(1, sizeof(size_t) * (gzip_payloads_count + 1));
+                            if (gzip_borders == NULL) {
+                                flb_errno();
+                                return -1;
+                            }
+                            if (flb_gzip_count(data, len, &gzip_borders, gzip_payloads_count) < 0) {
                                 flb_plg_error(ctx->ins,
                                              "failed to traverse boundaries of concatenated gzip payloads");
                                 return -1;

src/CMakeLists.txt

@@ -60,7 +60,6 @@ set(src
   flb_random.c
   flb_plugin.c
   flb_gzip.c
-  flb_gzip_concatenated.c
   flb_snappy.c
   flb_compression.c
   flb_http_common.c

src/flb_gzip.c

@@ -771,4 +771,68 @@ int flb_is_http_session_gzip_compressed(struct mk_http_session *session)
     }
 
     return gzip_compressed;
-}
+}
+
+static int vaild_os_flag(const char data)
+{
+    uint8_t p;
+
+    p = (uint8_t)data;
+    if (p == 0x00 || /* Fat Filesystem       */
+        p == 0x01 || /* Amiga                */
+        p == 0x02 || /* VMS                  */
+        p == 0x03 || /* Unix                 */
+        p == 0x04 || /* VM/CMS               */
+        p == 0x05 || /* Atari TOS            */
+        p == 0x06 || /* HPFS Filesystem (OS/2, NT) */
+        p == 0x07 || /* Macintosh            */
+        p == 0x08 || /* Z-System             */
+        p == 0x09 || /* CP/M                 */
+        p == 0x0a || /* TOPS-20              */
+        p == 0x0b || /* NTFS filesystem (NT) */
+        p == 0x0c || /* QDOS                 */
+        p == 0x0d || /* Acorn RISCOS         */
+        p == 0xff)   /* Unknown              */ {
+
+        return FLB_TRUE;
+    }
+
+    return FLB_FALSE;
+}
+
+size_t flb_gzip_count(const char *data, size_t len, size_t **out_borders, size_t border_count)
+{
+    int i;
+    size_t count = 0;
+    const uint8_t *p;
+    size_t *borders = NULL;
+
+    if (out_borders != NULL) {
+        borders = *out_borders;
+    }
+
+    p = (const uint8_t *) data;
+    /* search other gzip starting bits and method. */
+    for (i = 2; i < len &&
+                 i + 9 <= len; i++) {
+        /* A vaild gzip payloads are larger than 18 bytes. */
+        if (len - i < 18) {
+            break;
+        }
+
+        if (p[i] == 0x1F && p[i+1] == 0x8B && p[i+2] == 8 &&
+            vaild_os_flag(p[i+9])) {
+            if (out_borders != NULL) {
+                borders[count] = i;
+            }
+            count++;
+        }
+    }
+
+    if (out_borders != NULL && border_count >= count) {
+        /* The length of the last border refers to the original length. */
+        borders[border_count] = len;
+    }
+
+    return count;
+}

tests/internal/CMakeLists.txt

@@ -18,7 +18,6 @@ set(UNIT_TESTS_FILES
   http_client.c
   utils.c
   gzip.c
-  gzip_concatenated.c
   random.c
   config_map.c
   mp.c

tests/internal/gzip.c

@@ -40,7 +40,38 @@ void test_compress()
     flb_free(str);
 }
 
+void test_not_overflow_for_concatenated_gzip()
+{
+    const char data[] = {
+        0x00, 0x00, /* Initial padding */
+        0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* First gzip header (valid header) */
+        0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* Second gzip header (valid header) */
+        0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* Third gzip header (valid header) */
+    };
+    size_t len = sizeof(data);
+    size_t *borders = NULL;
+    size_t border_count = 0;
+    size_t count = 0;
+
+    /* Vaild gzip payloads have to 18 bytes lentgh at least.
+     * So, we get only 2 of vaild parts.
+     */
+    border_count = flb_gzip_count(data, len, NULL, 0);
+    TEST_CHECK(border_count == 2);
+
+    borders = (size_t *)flb_calloc(1, sizeof(size_t) * (border_count + 1));
+    TEST_CHECK(borders != NULL);
+
+    count = flb_gzip_count(data, len, &borders, border_count);
+    TEST_CHECK(count == 2);
+
+    if (borders != NULL) {
+        free(borders);
+    }
+}
+
 TEST_LIST = {
     {"compress", test_compress},
+    {"not_overflow", test_not_overflow_for_concatenated_gzip},
     { 0 }
 };