in_forward: Use full width of gzip header for checking whether concatenated or not

include/fluent-bit/flb_gzip.h

@@ -38,5 +38,6 @@ int flb_gzip_decompressor_dispatch(struct flb_decompression_context *context,
                                    void *out_data, size_t *out_size);
 
 int flb_is_http_session_gzip_compressed(struct mk_http_session *session);
+size_t flb_gzip_count(const char *data, size_t len, size_t **out_borders, size_t border_count);
 
 #endif

plugins/in_forward/fw_prot.c

@@ -1200,55 +1200,6 @@ int fw_prot_secure_forward_handshake(struct flb_input_instance *ins,
     return -1;
 }
 
-static size_t gzip_concatenated_count(const char *data, size_t len)
-{
-    int i;
-    size_t count = 0;
-    const uint8_t *p;
-
-    p = (const uint8_t *) data;
-
-    /* search other gzip starting bits and method. */
-    for (i = 2; i < len &&
-                 i + 2 <= len; i++) {
-        if (p[i] == 0x1F && p[i+1] == 0x8B && p[i+2] == 8) {
-            count++;
-        }
-    }
-
-    return count;
-}
-
-static size_t gzip_concatenated_borders(const char *data, size_t len, size_t **out_borders, size_t border_count)
-{
-    int i;
-    size_t count = 0;
-    const uint8_t *p;
-    size_t *borders = NULL;
-
-    p = (const uint8_t *) data;
-    borders = (size_t *) flb_calloc(1, sizeof(size_t) * (border_count + 1));
-    if (borders == NULL) {
-        flb_errno();
-        return -1;
-    }
-
-    /* search other gzip starting bits and method. */
-    for (i = 2; i < len &&
-                 i + 2 <= len; i++) {
-        if (p[i] == 0x1F && p[i+1] == 0x8B && p[i+2] == 8) {
-            borders[count] = i;
-            count++;
-        }
-    }
-    /* The length of the last border refers to the original length. */
-    borders[border_count] = len;
-
-    *out_borders = borders;
-
-    return count;
-}
-
 int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
 {
     int ret;
@@ -1543,11 +1494,16 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
                         size_t *gzip_borders = NULL;
                         const size_t original_len = len;
 
-                        gzip_payloads_count = gzip_concatenated_count(data, len);
+                        gzip_payloads_count = flb_gzip_count(data, len, NULL, 0);
                         flb_plg_debug(ctx->ins, "concatenated gzip payload count is %zd",
                                      gzip_payloads_count);
                         if (gzip_payloads_count > 0) {
-                            if (gzip_concatenated_borders(data, len, &gzip_borders, gzip_payloads_count) < 0) {
+                            gzip_borders = (size_t *)flb_calloc(1, sizeof(size_t) * (gzip_payloads_count + 1));
+                            if (gzip_borders == NULL) {
+                                flb_errno();
+                                return -1;
+                            }
+                            if (flb_gzip_count(data, len, &gzip_borders, gzip_payloads_count) < 0) {
                                 flb_plg_error(ctx->ins,
                                              "failed to traverse boundaries of concatenated gzip payloads");
                                 return -1;
@@ -1618,7 +1574,7 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
                         /* a valid payload of gzip is larger than 18 bytes. */
                         if (gzip_payloads_count > 0) {
                             if ((gzip_payloads_count - loop) > 0 &&
-                                (original_len - gzip_borders[loop]) > 18) {
+                                (original_len - gzip_borders[loop]) >= 18) {
                                 len = original_len - gzip_borders[loop];
                                 flb_plg_debug(ctx->ins, "left unconsumed %zd byte(s)", len);
                                 prev_pos = gzip_borders[loop];

src/flb_gzip.c

@@ -771,4 +771,68 @@ int flb_is_http_session_gzip_compressed(struct mk_http_session *session)
     }
 
     return gzip_compressed;
-}
+}
+
+static int vaild_os_flag(const char data)
+{
+    uint8_t p;
+
+    p = (uint8_t)data;
+    if (p == 0x00 || /* Fat Filesystem       */
+        p == 0x01 || /* Amiga                */
+        p == 0x02 || /* VMS                  */
+        p == 0x03 || /* Unix                 */
+        p == 0x04 || /* VM/CMS               */
+        p == 0x05 || /* Atari TOS            */
+        p == 0x06 || /* HPFS Filesystem (OS/2, NT) */
+        p == 0x07 || /* Macintosh            */
+        p == 0x08 || /* Z-System             */
+        p == 0x09 || /* CP/M                 */
+        p == 0x0a || /* TOPS-20              */
+        p == 0x0b || /* NTFS filesystem (NT) */
+        p == 0x0c || /* QDOS                 */
+        p == 0x0d || /* Acorn RISCOS         */
+        p == 0xff)   /* Unknown              */ {
+
+        return FLB_TRUE;
+    }
+
+    return FLB_FALSE;
+}
+
+size_t flb_gzip_count(const char *data, size_t len, size_t **out_borders, size_t border_count)
+{
+    int i;
+    size_t count = 0;
+    const uint8_t *p;
+    size_t *borders = NULL;
+
+    if (out_borders != NULL) {
+        borders = *out_borders;
+    }
+
+    p = (const uint8_t *) data;
+    /* search other gzip starting bits and method. */
+    for (i = 2; i < len &&
+                 i + 9 <= len; i++) {
+        /* A vaild gzip payloads are larger than 18 bytes. */
+        if (len - i < 18) {
+            break;
+        }
+
+        if (p[i] == 0x1F && p[i+1] == 0x8B && p[i+2] == 8 &&
+            vaild_os_flag(p[i+9])) {
+            if (out_borders != NULL) {
+                borders[count] = i;
+            }
+            count++;
+        }
+    }
+
+    if (out_borders != NULL && border_count >= count) {
+        /* The length of the last border refers to the original length. */
+        borders[border_count] = len;
+    }
+
+    return count;
+}

tests/internal/gzip.c

@@ -40,7 +40,69 @@ void test_compress()
     flb_free(str);
 }
 
+void test_concatenated_gzip_count()
+{
+     int ret;
+    int sample_len;
+    char *in_data = morpheus;
+    size_t in_len;
+    void *str;
+    size_t len;
+    flb_sds_t payload = NULL;
+    flb_sds_t payload2 = NULL;
+    size_t border_count = 0;
+
+    sample_len = strlen(morpheus);
+    in_len = sample_len;
+    ret = flb_gzip_compress(in_data, in_len, &str, &len);
+    TEST_CHECK(ret == 0);
+
+    payload = flb_sds_create_len((char *)str, len);
+    payload2 = flb_sds_create_len((char *)str, len);
+    ret = flb_sds_cat_safe(&payload, payload2, flb_sds_len(payload2));
+    TEST_CHECK(ret == 0);
+
+    border_count = flb_gzip_count(payload, flb_sds_len(payload), NULL, 0);
+    TEST_CHECK(border_count == 1);
+
+    flb_free(str);
+    flb_sds_destroy(payload);
+    flb_sds_destroy(payload2);
+}
+
+void test_not_overflow_for_concatenated_gzip()
+{
+    const char data[] = {
+        0x00, 0x00, /* Initial padding */
+        0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* First gzip header (valid header) */
+        0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* Second gzip header (valid header) */
+        0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* Third gzip header (valid header) */
+    };
+    size_t len = sizeof(data);
+    size_t *borders = NULL;
+    size_t border_count = 0;
+    size_t count = 0;
+
+    /* Vaild gzip payloads have to 18 bytes lentgh at least.
+     * So, we get only 2 of vaild parts.
+     */
+    border_count = flb_gzip_count(data, len, NULL, 0);
+    TEST_CHECK(border_count == 2);
+
+    borders = (size_t *)flb_calloc(1, sizeof(size_t) * (border_count + 1));
+    TEST_CHECK(borders != NULL);
+
+    count = flb_gzip_count(data, len, &borders, border_count);
+    TEST_CHECK(count == 2);
+
+    if (borders != NULL) {
+        free(borders);
+    }
+}
+
 TEST_LIST = {
     {"compress", test_compress},
+    {"count",  test_concatenated_gzip_count},
+    {"not_overflow", test_not_overflow_for_concatenated_gzip},
     { 0 }
 };