From a2cf70d7a9e739d64541ef3d4cbda2555b0a8f38 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek <58699848+sumo-drosiek@users.noreply.github.com> Date: Fri, 28 Jun 2024 11:00:16 +0200 Subject: [PATCH] fix: join multiparts on file beginning (#3784) * fix: join multiparts on file beginning Signed-off-by: Dominik Rosiek * chore: changelog Signed-off-by: Dominik Rosiek * Apply suggestions from code review * Apply suggestions from code review --------- Signed-off-by: Dominik Rosiek (cherry picked from commit 58f19f8c38433f4a0f176c76cacd2fa5571261e9) --- .changelog/3784.fixed.txt | 1 + .../conf/logs/collector/common/filelog_receiver.yaml | 6 ++++-- .../conf/logs/collector/otelcloudwatch/config.yaml | 3 ++- tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml | 4 ++-- tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml | 4 ++-- .../logs_otc_daemonset/multiple_multiline.output.yaml | 4 ++-- .../goldenfile/logs_otc_windows/configmap/basic.output.yaml | 4 ++-- .../goldenfile/logs_otc_windows/configmap/debug.output.yaml | 4 ++-- .../daemonset/multiple_multiline.output.yaml | 4 ++-- 9 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 .changelog/3784.fixed.txt diff --git a/.changelog/3784.fixed.txt b/.changelog/3784.fixed.txt new file mode 100644 index 0000000000..78347b1f77 --- /dev/null +++ b/.changelog/3784.fixed.txt @@ -0,0 +1 @@ +fix: join multiparts on file beginning \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml b/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml index 2dbc0947cb..db47cc9b1a 100644 --- a/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml +++ b/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml @@ -99,7 +99,8 @@ filelog/containers: output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine - max_unmatched_batch_size: 1 + ## Ensure we combine everything up to `is_last_entry` even on the file beginning + max_unmatched_batch_size: 0 ## merge-cri-lines stitches back together log lines split by CRI logging drivers. ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "logtag": "P" } @@ -113,7 +114,8 @@ filelog/containers: overwrite_with: newest source_identifier: attributes["log.file.path"] type: recombine - max_unmatched_batch_size: 1 + ## Ensure we combine everything up to `is_last_entry` even on the file beginning + max_unmatched_batch_size: 0 ## strip-trailing-newline removes the trailing "\n" from the `log` key. This is required for logs coming from Docker container runtime. ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" } diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml index 68dafd11ea..f3c5cf64b8 100644 --- a/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml +++ b/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml @@ -61,7 +61,8 @@ processors: overwrite_with: newest source_identifier: resource["cloudwatch.log.stream"] type: recombine - max_unmatched_batch_size: 1 + ## Ensure we combine everything up to `is_last_entry` even on the file beginning + max_unmatched_batch_size: 0 - id: merge-multiline-logs combine_field: attributes.log combine_with: "\n" diff --git a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml index 1d955c4764..365038aff5 100644 --- a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml @@ -99,7 +99,7 @@ data: combine_with: "" id: merge-docker-lines is_last_entry: body.log matches "\n$" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine @@ -107,7 +107,7 @@ data: combine_with: "" id: merge-cri-lines is_last_entry: body.logtag == "F" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: extract-metadata-from-filepath overwrite_with: newest source_identifier: attributes["log.file.path"] diff --git a/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml index 9a411c84cf..c7d0cb71fa 100644 --- a/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml @@ -109,7 +109,7 @@ data: combine_with: "" id: merge-docker-lines is_last_entry: body.log matches "\n$" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine @@ -117,7 +117,7 @@ data: combine_with: "" id: merge-cri-lines is_last_entry: body.logtag == "F" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: extract-metadata-from-filepath overwrite_with: newest source_identifier: attributes["log.file.path"] diff --git a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml index 271bdef1ae..5f206747f7 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml @@ -99,7 +99,7 @@ data: combine_with: "" id: merge-docker-lines is_last_entry: body.log matches "\n$" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine @@ -107,7 +107,7 @@ data: combine_with: "" id: merge-cri-lines is_last_entry: body.logtag == "F" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: extract-metadata-from-filepath overwrite_with: newest source_identifier: attributes["log.file.path"] diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml index 2add6b0b4e..985b1e1285 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml @@ -78,7 +78,7 @@ data: combine_with: "" id: merge-docker-lines is_last_entry: body.log matches "\n$" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine @@ -86,7 +86,7 @@ data: combine_with: "" id: merge-cri-lines is_last_entry: body.logtag == "F" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: extract-metadata-from-filepath overwrite_with: newest source_identifier: attributes["log.file.path"] diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml index 033e6d97c2..366b17c151 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml @@ -89,7 +89,7 @@ data: combine_with: "" id: merge-docker-lines is_last_entry: body.log matches "\n$" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine @@ -97,7 +97,7 @@ data: combine_with: "" id: merge-cri-lines is_last_entry: body.logtag == "F" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: extract-metadata-from-filepath overwrite_with: newest source_identifier: attributes["log.file.path"] diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml index 2199bf019a..e7eb6e5f41 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml @@ -78,7 +78,7 @@ data: combine_with: "" id: merge-docker-lines is_last_entry: body.log matches "\n$" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine @@ -86,7 +86,7 @@ data: combine_with: "" id: merge-cri-lines is_last_entry: body.logtag == "F" - max_unmatched_batch_size: 1 + max_unmatched_batch_size: 0 output: extract-metadata-from-filepath overwrite_with: newest source_identifier: attributes["log.file.path"]