Skip to content

Commit

Permalink
in_forward: gzip: Use full width of gzip header for checking whether …
Browse files Browse the repository at this point in the history 
…concatenated or not

Using for concatenated gzip conformation with magic bytes, compression
method and OS flags.

* 0x1f & 0x8b
* 8 (deflate)
* skip 7 bytes
* OS flags

Signed-off-by: Hiroshi Hatake <[email protected]>
  • Loading branch information
@cosmo0920
cosmo0920 committed Aug 3, 2024
1 parent c767acf commit cad1bea
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 53 deletions.
1 change: 1 addition & 0 deletions include/fluent-bit/flb_gzip.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,6 @@ int flb_gzip_decompressor_dispatch(struct flb_decompression_context *context,
void *out_data, size_t *out_size);

int flb_is_http_session_gzip_compressed(struct mk_http_session *session);
size_t flb_gzip_count(const char *data, size_t len, size_t **out_borders, size_t border_count);

#endif
60 changes: 8 additions & 52 deletions plugins/in_forward/fw_prot.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1200,55 +1200,6 @@ int fw_prot_secure_forward_handshake(struct flb_input_instance *ins,
return -1;
}

static size_t gzip_concatenated_count(const char *data, size_t len)
{
int i;
size_t count = 0;
const uint8_t *p;

p = (const uint8_t *) data;

/* search other gzip starting bits and method. */
for (i = 2; i < len &&
i + 2 <= len; i++) {
if (p[i] == 0x1F && p[i+1] == 0x8B && p[i+2] == 8) {
count++;
}
}

return count;
}

static size_t gzip_concatenated_borders(const char *data, size_t len, size_t **out_borders, size_t border_count)
{
int i;
size_t count = 0;
const uint8_t *p;
size_t *borders = NULL;

p = (const uint8_t *) data;
borders = (size_t *) flb_calloc(1, sizeof(size_t) * (border_count + 1));
if (borders == NULL) {
flb_errno();
return -1;
}

/* search other gzip starting bits and method. */
for (i = 2; i < len &&
i + 2 <= len; i++) {
if (p[i] == 0x1F && p[i+1] == 0x8B && p[i+2] == 8) {
borders[count] = i;
count++;
}
}
/* The length of the last border refers to the original length. */
borders[border_count] = len;

*out_borders = borders;

return count;
}

int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
{
int ret;
Expand Down Expand Up @@ -1543,11 +1494,16 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
size_t *gzip_borders = NULL;
const size_t original_len = len;

gzip_payloads_count = gzip_concatenated_count(data, len);
gzip_payloads_count = flb_gzip_count(data, len, NULL, 0);
flb_plg_debug(ctx->ins, "concatenated gzip payload count is %zd",
gzip_payloads_count);
if (gzip_payloads_count > 0) {
if (gzip_concatenated_borders(data, len, &gzip_borders, gzip_payloads_count) < 0) {
gzip_borders = (size_t *)flb_calloc(1, sizeof(size_t) * (gzip_payloads_count + 1));
if (gzip_borders == NULL) {
flb_errno();
return -1;
}
if (flb_gzip_count(data, len, &gzip_borders, gzip_payloads_count) < 0) {
flb_plg_error(ctx->ins,
"failed to traverse boundaries of concatenated gzip payloads");
return -1;
Expand Down Expand Up @@ -1618,7 +1574,7 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
/* a valid payload of gzip is larger than 18 bytes. */
if (gzip_payloads_count > 0) {
if ((gzip_payloads_count - loop) > 0 &&
(original_len - gzip_borders[loop]) > 18) {
(original_len - gzip_borders[loop]) >= 18) {
len = original_len - gzip_borders[loop];
flb_plg_debug(ctx->ins, "left unconsumed %zd byte(s)", len);
prev_pos = gzip_borders[loop];
Expand Down
66 changes: 65 additions & 1 deletion src/flb_gzip.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -771,4 +771,68 @@ int flb_is_http_session_gzip_compressed(struct mk_http_session *session)
}

return gzip_compressed;
}
}

static int vaild_os_flag(const char data)
{
uint8_t p;

p = (uint8_t)data;
if (p == 0x00 || /* Fat Filesystem */
p == 0x01 || /* Amiga */
p == 0x02 || /* VMS */
p == 0x03 || /* Unix */
p == 0x04 || /* VM/CMS */
p == 0x05 || /* Atari TOS */
p == 0x06 || /* HPFS Filesystem (OS/2, NT) */
p == 0x07 || /* Macintosh */
p == 0x08 || /* Z-System */
p == 0x09 || /* CP/M */
p == 0x0a || /* TOPS-20 */
p == 0x0b || /* NTFS filesystem (NT) */
p == 0x0c || /* QDOS */
p == 0x0d || /* Acorn RISCOS */
p == 0xff) /* Unknown */ {

return FLB_TRUE;
}

return FLB_FALSE;
}

size_t flb_gzip_count(const char *data, size_t len, size_t **out_borders, size_t border_count)
{
int i;
size_t count = 0;
const uint8_t *p;
size_t *borders = NULL;

if (out_borders != NULL) {
borders = *out_borders;
}

p = (const uint8_t *) data;
/* search other gzip starting bits and method. */
for (i = 2; i < len &&
i + 9 <= len; i++) {
/* A vaild gzip payloads are larger than 18 bytes. */
if (len - i < 18) {
break;
}

if (p[i] == 0x1F && p[i+1] == 0x8B && p[i+2] == 8 &&
vaild_os_flag(p[i+9])) {
if (out_borders != NULL) {
borders[count] = i;
}
count++;
}
}

if (out_borders != NULL && border_count >= count) {
/* The length of the last border refers to the original length. */
borders[border_count] = len;
}

return count;
}
62 changes: 62 additions & 0 deletions tests/internal/gzip.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,69 @@ void test_compress()
flb_free(str);
}

void test_concatenated_gzip_count()
{
int ret;
int sample_len;
char *in_data = morpheus;
size_t in_len;
void *str;
size_t len;
flb_sds_t payload = NULL;
flb_sds_t payload2 = NULL;
size_t border_count = 0;

sample_len = strlen(morpheus);
in_len = sample_len;
ret = flb_gzip_compress(in_data, in_len, &str, &len);
TEST_CHECK(ret == 0);

payload = flb_sds_create_len((char *)str, len);
payload2 = flb_sds_create_len((char *)str, len);
ret = flb_sds_cat_safe(&payload, payload2, flb_sds_len(payload2));
TEST_CHECK(ret == 0);

border_count = flb_gzip_count(payload, flb_sds_len(payload), NULL, 0);
TEST_CHECK(border_count == 1);

flb_free(str);
flb_sds_destroy(payload);
flb_sds_destroy(payload2);
}

void test_not_overflow_for_concatenated_gzip()
{
const char data[] = {
0x00, 0x00, /* Initial padding */
0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* First gzip header (valid header) */
0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* Second gzip header (valid header) */
0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, /* Third gzip header (valid header) */
};
size_t len = sizeof(data);
size_t *borders = NULL;
size_t border_count = 0;
size_t count = 0;

/* Vaild gzip payloads have to 18 bytes lentgh at least.
* So, we get only 2 of vaild parts.
*/
border_count = flb_gzip_count(data, len, NULL, 0);
TEST_CHECK(border_count == 2);

borders = (size_t *)flb_calloc(1, sizeof(size_t) * (border_count + 1));
TEST_CHECK(borders != NULL);

count = flb_gzip_count(data, len, &borders, border_count);
TEST_CHECK(count == 2);

if (borders != NULL) {
free(borders);
}
}

TEST_LIST = {
{"compress", test_compress},
{"count", test_concatenated_gzip_count},
{"not_overflow", test_not_overflow_for_concatenated_gzip},
{ 0 }
};

0 comments on commit cad1bea

Please sign in to comment.