From b4592c171ad025ecbfd41dd76fad760b52919290 Mon Sep 17 00:00:00 2001 From: braydonk Date: Sat, 28 Oct 2023 15:49:37 +0000 Subject: [PATCH] parser: add Time_System_Timezone Signed-off-by: braydonk --- include/fluent-bit/flb_parser.h | 11 +- .../in_kubernetes_events/kubernetes_events.c | 2 +- src/flb_parser.c | 25 ++- src/flb_parser_json.c | 2 +- src/flb_parser_logfmt.c | 2 +- src/flb_parser_ltsv.c | 2 +- src/flb_parser_regex.c | 2 +- src/multiline/flb_ml_parser_cri.c | 1 + src/multiline/flb_ml_parser_docker.c | 1 + tests/internal/fuzzers/engine_fuzzer.c | 2 +- tests/internal/fuzzers/parse_json_fuzzer.c | 2 +- tests/internal/fuzzers/parse_logfmt_fuzzer.c | 4 +- tests/internal/fuzzers/parse_ltsv_fuzzer.c | 4 +- tests/internal/fuzzers/parser_fuzzer.c | 2 +- tests/internal/parser.c | 2 +- tests/internal/parser_json.c | 12 +- tests/internal/parser_logfmt.c | 8 +- tests/internal/parser_ltsv.c | 10 +- tests/internal/parser_regex.c | 10 +- tests/runtime/filter_parser.c | 149 ++++++++++++++++-- 20 files changed, 205 insertions(+), 48 deletions(-) diff --git a/include/fluent-bit/flb_parser.h b/include/fluent-bit/flb_parser.h index 4f136b9a5c1..0810ed000d3 100644 --- a/include/fluent-bit/flb_parser.h +++ b/include/fluent-bit/flb_parser.h @@ -48,6 +48,7 @@ struct flb_parser { char *time_fmt_full; /* original given time format */ char *time_key; /* field name that contains the time */ int time_offset; /* fixed UTC offset */ + int time_system_timezone; /* use the system timezone as a fallback */ int time_keep; /* keep time field */ int time_strict; /* parse time field strictly */ int logfmt_no_bare_keys; /* in logfmt parsers, require all keys to have values */ @@ -74,13 +75,18 @@ enum { FLB_PARSER_TYPE_HEX, }; -static inline time_t flb_parser_tm2time(const struct flb_tm *src) +static inline time_t flb_parser_tm2time(const struct flb_tm *src, + int use_system_timezone) { struct tm tmp; time_t res; tmp = src->tm; - res = timegm(&tmp) - flb_tm_gmtoff(src); + if (use_system_timezone) { + res = mktime(&tmp); + } else { + res = timegm(&tmp) - flb_tm_gmtoff(src); + } return res; } @@ -92,6 +98,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, const char *time_offset, int time_keep, int time_strict, + int time_system_timezone, int logfmt_no_bare_keys, struct flb_parser_types *types, int types_len, diff --git a/plugins/in_kubernetes_events/kubernetes_events.c b/plugins/in_kubernetes_events/kubernetes_events.c index 6231becb0ad..40bf5c90acd 100644 --- a/plugins/in_kubernetes_events/kubernetes_events.c +++ b/plugins/in_kubernetes_events/kubernetes_events.c @@ -225,7 +225,7 @@ static int record_get_field_time(msgpack_object *obj, const char *fieldname, str return -2; } - val->tm.tv_sec = flb_parser_tm2time(&tm); + val->tm.tv_sec = flb_parser_tm2time(&tm, FLB_FALSE); val->tm.tv_nsec = 0; return 0; diff --git a/src/flb_parser.c b/src/flb_parser.c index dc78f076fda..8df7f56dfb5 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -152,6 +152,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, const char *time_offset, int time_keep, int time_strict, + int time_system_timezone, int logfmt_no_bare_keys, struct flb_parser_types *types, int types_len, @@ -312,8 +313,17 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, p->time_frac_secs = (tmp + 2); } - /* Optional fixed timezone offset */ - if (time_offset) { + /* + * Fall back to the system timezone + * if there is no zone parsed from the log. + */ + p->time_system_timezone = time_system_timezone; + + /* + * Optional fixed timezone offset, only applied if + * not falling back to system timezone. + */ + if (!p->time_system_timezone && time_offset) { diff = 0; len = strlen(time_offset); ret = flb_parser_tzone_offset(time_offset, len, &diff); @@ -487,6 +497,7 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, int skip_empty; int time_keep; int time_strict; + int time_system_timezone; int logfmt_no_bare_keys; int types_len; struct mk_list *head; @@ -561,6 +572,13 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, flb_sds_destroy(tmp_str); } + time_system_timezone = FLB_FALSE; + tmp_str = get_parser_key(config, cf, s, "time_system_timezone"); + if (tmp_str) { + time_system_timezone = flb_utils_bool(tmp_str); + flb_sds_destroy(tmp_str); + } + /* time_offset (UTC offset) */ time_offset = get_parser_key(config, cf, s, "time_offset"); @@ -587,7 +605,8 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf, /* Create the parser context */ if (!flb_parser_create(name, format, regex, skip_empty, time_fmt, time_key, time_offset, time_keep, time_strict, - logfmt_no_bare_keys, types, types_len, decoders, config)) { + time_system_timezone, logfmt_no_bare_keys, types, types_len, + decoders, config)) { goto fconf_error; } diff --git a/src/flb_parser_json.c b/src/flb_parser_json.c index d8552cd7233..1693f766606 100644 --- a/src/flb_parser_json.c +++ b/src/flb_parser_json.c @@ -207,7 +207,7 @@ int flb_parser_json_do(struct flb_parser *parser, skip = map_size; } else { - time_lookup = flb_parser_tm2time(&tm); + time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); } /* Compose a new map without the time_key field */ diff --git a/src/flb_parser_logfmt.c b/src/flb_parser_logfmt.c index 6d88cd6a0e7..12de008230a 100644 --- a/src/flb_parser_logfmt.c +++ b/src/flb_parser_logfmt.c @@ -166,7 +166,7 @@ static int logfmt_parser(struct flb_parser *parser, parser->name, parser->time_fmt_full); return -1; } - *time_lookup = flb_parser_tm2time(&tm); + *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); } time_found = FLB_TRUE; } diff --git a/src/flb_parser_ltsv.c b/src/flb_parser_ltsv.c index dae458c9a4a..297f528515f 100644 --- a/src/flb_parser_ltsv.c +++ b/src/flb_parser_ltsv.c @@ -139,7 +139,7 @@ static int ltsv_parser(struct flb_parser *parser, parser->name, parser->time_fmt_full); return -1; } - *time_lookup = flb_parser_tm2time(&tm); + *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); } time_found = FLB_TRUE; } diff --git a/src/flb_parser_regex.c b/src/flb_parser_regex.c index 7982c7d4a42..5dc4a965a75 100644 --- a/src/flb_parser_regex.c +++ b/src/flb_parser_regex.c @@ -87,7 +87,7 @@ static void cb_results(const char *name, const char *value, } pcb->time_frac = frac; - pcb->time_lookup = flb_parser_tm2time(&tm); + pcb->time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); if (parser->time_keep == FLB_FALSE) { pcb->num_skipped++; diff --git a/src/multiline/flb_ml_parser_cri.c b/src/multiline/flb_ml_parser_cri.c index 1a4ac9bbc5a..e4bd6a0e0f3 100644 --- a/src/multiline/flb_ml_parser_cri.c +++ b/src/multiline/flb_ml_parser_cri.c @@ -40,6 +40,7 @@ static struct flb_parser *cri_parser_create(struct flb_config *config) NULL, /* time offset */ FLB_TRUE, /* time keep */ FLB_FALSE, /* time strict */ + FLB_FALSE, /* time system timezone */ FLB_FALSE, /* no bare keys */ NULL, /* parser types */ 0, /* types len */ diff --git a/src/multiline/flb_ml_parser_docker.c b/src/multiline/flb_ml_parser_docker.c index 0573affab33..51550f3893e 100644 --- a/src/multiline/flb_ml_parser_docker.c +++ b/src/multiline/flb_ml_parser_docker.c @@ -35,6 +35,7 @@ static struct flb_parser *docker_parser_create(struct flb_config *config) NULL, /* time offset */ FLB_TRUE, /* time keep */ FLB_FALSE, /* time strict */ + FLB_FALSE, /* time system timezone */ FLB_FALSE, /* no bare keys */ NULL, /* parser types */ 0, /* types len */ diff --git a/tests/internal/fuzzers/engine_fuzzer.c b/tests/internal/fuzzers/engine_fuzzer.c index 498b86a8fcc..5d7e9ab802b 100644 --- a/tests/internal/fuzzers/engine_fuzzer.c +++ b/tests/internal/fuzzers/engine_fuzzer.c @@ -143,7 +143,7 @@ int LLVMFuzzerInitialize(int *argc, char ***argv) { parser = flb_parser_create("timestamp", "regex", "^(?