How to debug failing operators?

[davispuh]

Consider I've receiver with bunch of operators like

receivers:
  journald:
    operators:
      - type: copy
        from: body.MESSAGE
        to: resource['log.message']
      - if: body.MESSAGE[0:1] == '{'
        type: json_parser
        parse_from: body.MESSAGE
        parse_to: body.info
      - if: 'body.MESSAGE matches "^[\\d+/ :]+ \\[\\w+\\] \\d+#\\d+: "'
        type: regex_parser
        parse_from: body.MESSAGE
        parse_to: body.info
      # - and so on

And I have some logs that causes these to fail and hence that log message is not delivered. How can I find out what log message caused issue?

In current logs I only see

{
    "level": "error",
    "ts": "2025-02-12T20:04:08.585Z",
    "caller": "helper/transformer.go:100",
    "msg": "Failed to process entry",
    "kind": "receiver",
    "name": "journald",
    "data_type": "logs",
    "operator_id": "regex_parser",
    "operator_type": "regex_parser",
    "error": "running if expr: interface conversion: interface {} is []interface {}, not string (1:14)\n | body.MESSAGE matches \"^[\\\\d+/ :]+ \\\\[\\\\w+\\\\] \\\\d+#\\\\d+: \"\n | .............^",
    "action": "send",
    "stacktrace": "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).HandleEntryError\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:100\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWithCallback\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:105\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:98\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/regex.(*Parser).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/parser/regex/parser.go:31\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/severity.(*Parser).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/parser/severity/parser.go:21\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWithCallback\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:108\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:98\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/json.(*Parser).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/parser/json/parser.go:26\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:86\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:86\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:86\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:86\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:86\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:86\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/input/journald.(*Input).runJournalctl.func2\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/input/journald/input.go:196"
}

Which doesn't tell me what log line looked like that caused this failure so I've no idea how to fix it.

Some more failure examples

{
    "level": "error",
    "ts": "2025-02-12T20:52:03.865Z",
    "caller": "journald/input.go:197",
    "msg": "failed to write entry",
    "kind": "receiver",
    "name": "journald",
    "data_type": "logs",
    "operator_id": "journald_input",
    "operator_type": "journald_input",
    "error": "invalid character ',' looking for beginning of value",
    "stacktrace": "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/input/journald.(*Input).runJournalctl.func2\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/input/journald/input.go:197"
}
{
    "level": "error",
    "ts": "2025-02-12T20:52:03.865Z",
    "caller": "helper/transformer.go:100",
    "msg": "Failed to process entry",
    "kind": "receiver",
    "name": "journald",
    "data_type": "logs",
    "operator_id": "json_parser",
    "operator_type": "json_parser",
    "error": "invalid character ',' looking for beginning of value",
    "action": "send",
    "stacktrace": "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).HandleEntryError\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:100\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ParseWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:142\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWithCallback\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:111\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:98\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/json.(*Parser).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/parser/json/parser.go:26\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:86\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/input/journald.(*Input).runJournalctl.func2\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/input/journald/input.go:196"
}
{
    "level": "error",
    "ts": "2025-02-12T20:55:36.866Z",
    "caller": "journald/input.go:197",
    "msg": "failed to write entry",
    "kind": "receiver",
    "name": "journald",
    "data_type": "logs",
    "operator_id": "journald_input",
    "operator_type": "journald_input",
    "error": "json: string unexpected end of JSON input",
    "stacktrace": "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/input/journald.(*Input).runJournalctl.func2\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/input/journald/input.go:197"
}
{
    "level": "error",
    "ts": "2025-02-12T20:55:36.868Z",
    "caller": "helper/transformer.go:100",
    "msg": "Failed to process entry",
    "kind": "receiver",
    "name": "journald",
    "data_type": "logs",
    "operator_id": "json_parser",
    "operator_type": "json_parser",
    "error": "json: string unexpected end of JSON input",
    "action": "send",
    "stacktrace": "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).HandleEntryError\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:100\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ParseWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:142\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWithCallback\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:111\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*ParserOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/parser.go:98\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/json.(*Parser).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/parser/json/parser.go:26\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:86\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*TransformerOperator).ProcessWith\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/transformer.go:92\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/transformer/copy.(*Transformer).Process\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/transformer/copy/transformer.go:23\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*WriterOperator).Write\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/helper/writer.go:54\ngithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/input/journald.(*Input).runJournalctl.func2\n\tgithub.com/open-telemetry/opentelemetry-collector-contrib/pkg/[email protected]/operator/input/journald/input.go:196"
}