From 8b581be2869c5044bcf8113d12172c864c34cfe1 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek <58699848+sumo-drosiek@users.noreply.github.com> Date: Fri, 28 Jun 2024 11:00:16 +0200 Subject: [PATCH] fix: join multiparts on file beginning (#3784) * fix: join multiparts on file beginning Signed-off-by: Dominik Rosiek * chore: changelog Signed-off-by: Dominik Rosiek * Apply suggestions from code review * Apply suggestions from code review --------- Signed-off-by: Dominik Rosiek (cherry picked from commit 58f19f8c38433f4a0f176c76cacd2fa5571261e9) --- .changelog/3787.fixed.txt | 1 + .../sumologic/conf/logs/collector/otelcloudwatch/config.yaml | 1 + deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml | 2 ++ tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml | 2 ++ .../logs_otc_daemonset/multiple_multiline.output.yaml | 2 ++ 5 files changed, 8 insertions(+) create mode 100644 .changelog/3787.fixed.txt diff --git a/.changelog/3787.fixed.txt b/.changelog/3787.fixed.txt new file mode 100644 index 0000000000..78347b1f77 --- /dev/null +++ b/.changelog/3787.fixed.txt @@ -0,0 +1 @@ +fix: join multiparts on file beginning \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml index a0864018d2..1c3f6e0602 100644 --- a/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml +++ b/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml @@ -61,6 +61,7 @@ processors: overwrite_with: newest source_identifier: resource["cloudwatch.log.stream"] type: recombine + max_unmatched_batch_size: 0 - id: merge-multiline-logs combine_field: attributes.log combine_with: "\n" diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml index 42b155f2f4..df05790fb3 100644 --- a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml @@ -122,6 +122,7 @@ receivers: output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine + max_unmatched_batch_size: 0 ## merge-cri-lines stitches back together log lines split by CRI logging drivers. ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "logtag": "P" } @@ -135,6 +136,7 @@ receivers: overwrite_with: newest source_identifier: attributes["log.file.path"] type: recombine + max_unmatched_batch_size: 0 ## strip-trailing-newline removes the trailing "\n" from the `log` key. This is required for logs coming from Docker container runtime. ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" } diff --git a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml index 30237193e9..36060a738f 100644 --- a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml @@ -97,6 +97,7 @@ data: combine_with: "" id: merge-docker-lines is_last_entry: body.log matches "\n$" + max_unmatched_batch_size: 0 output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine @@ -104,6 +105,7 @@ data: combine_with: "" id: merge-cri-lines is_last_entry: body.logtag == "F" + max_unmatched_batch_size: 0 output: extract-metadata-from-filepath overwrite_with: newest source_identifier: attributes["log.file.path"] diff --git a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml index af063b3b57..db7e54699d 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml @@ -97,6 +97,7 @@ data: combine_with: "" id: merge-docker-lines is_last_entry: body.log matches "\n$" + max_unmatched_batch_size: 0 output: strip-trailing-newline source_identifier: attributes["log.file.path"] type: recombine @@ -104,6 +105,7 @@ data: combine_with: "" id: merge-cri-lines is_last_entry: body.logtag == "F" + max_unmatched_batch_size: 0 output: extract-metadata-from-filepath overwrite_with: newest source_identifier: attributes["log.file.path"]