From 30fcb06b310df1ff65eec6dc3a9d4f092771ce9e Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Thu, 10 Oct 2024 12:46:26 +0200 Subject: [PATCH] fix: filter ES error responses --- appender_test.go | 18 +++++++++++------- bulk_indexer.go | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/appender_test.go b/appender_test.go index 7c03740..4a882b3 100644 --- a/appender_test.go +++ b/appender_test.go @@ -748,7 +748,7 @@ func TestAppenderIndexFailedLogging(t *testing.T) { for i, item := range result.Items { itemResp := item["create"] itemResp.Index = "an_index" - switch i % 4 { + switch i % 5 { case 0: itemResp.Error.Type = "error_type" itemResp.Error.Reason = "error_reason_even. Preview of field's value: 'abc def ghi'" @@ -761,7 +761,9 @@ func TestAppenderIndexFailedLogging(t *testing.T) { case 3: itemResp.Error.Type = "x_content_parse_exception" itemResp.Error.Reason = "this reason should not be logged" - + case 4: + itemResp.Error.Type = "document_parsing_exception" + itemResp.Error.Reason = "this reason should not be logged" } item["create"] = itemResp } @@ -777,7 +779,7 @@ func TestAppenderIndexFailedLogging(t *testing.T) { require.NoError(t, err) defer indexer.Close(context.Background()) - const N = 4 * 2 + const N = 5 * 2 for i := 0; i < N; i++ { addMinimalDoc(t, indexer, "logs-foo-testing") } @@ -789,14 +791,16 @@ func TestAppenderIndexFailedLogging(t *testing.T) { return entries[i].Message < entries[j].Message }) require.Len(t, entries, N/2) - assert.Equal(t, "failed to index documents in 'an_index' (error_type): error_reason_even", entries[0].Message) + assert.Equal(t, "failed to index documents in 'an_index' (document_parsing_exception): ", entries[0].Message) assert.Equal(t, int64(2), entries[0].Context[0].Integer) - assert.Equal(t, "failed to index documents in 'an_index' (error_type): error_reason_odd", entries[1].Message) + assert.Equal(t, "failed to index documents in 'an_index' (error_type): error_reason_even", entries[1].Message) assert.Equal(t, int64(2), entries[1].Context[0].Integer) - assert.Equal(t, "failed to index documents in 'an_index' (unavailable_shards_exception): ", entries[2].Message) + assert.Equal(t, "failed to index documents in 'an_index' (error_type): error_reason_odd", entries[2].Message) assert.Equal(t, int64(2), entries[2].Context[0].Integer) - assert.Equal(t, "failed to index documents in 'an_index' (x_content_parse_exception): ", entries[3].Message) + assert.Equal(t, "failed to index documents in 'an_index' (unavailable_shards_exception): ", entries[3].Message) assert.Equal(t, int64(2), entries[3].Context[0].Integer) + assert.Equal(t, "failed to index documents in 'an_index' (x_content_parse_exception): ", entries[4].Message) + assert.Equal(t, int64(2), entries[4].Context[0].Integer) } func TestAppenderRetryLimit(t *testing.T) { diff --git a/bulk_indexer.go b/bulk_indexer.go index 3dacde9..6b6a4dd 100644 --- a/bulk_indexer.go +++ b/bulk_indexer.go @@ -158,7 +158,7 @@ func init() { return true }) // For specific exceptions, remove item.Error.Reason as it may contain sensitive request content. - if item.Error.Type == "unavailable_shards_exception" || item.Error.Type == "x_content_parse_exception" { + if item.Error.Type == "unavailable_shards_exception" || item.Error.Type == "x_content_parse_exception" || item.Error.Type == "document_parsing_exception" { item.Error.Reason = "" }