From 68c4a642a7e9e38232999f8ac17f77c92ec40929 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 13 Dec 2024 18:44:04 -0500 Subject: [PATCH] Fix more tests --- filebeat/input/filestream/environment_test.go | 1 + .../filestream/input_integration_test.go | 24 ++-- .../legacy_metrics_integration_test.go | 8 +- .../filestream/metrics_integration_test.go | 12 +- .../filestream/parsers_integration_test.go | 110 +++++++++++------- 5 files changed, 97 insertions(+), 58 deletions(-) diff --git a/filebeat/input/filestream/environment_test.go b/filebeat/input/filestream/environment_test.go index 58e048b4b40f..80460d6b3b4a 100644 --- a/filebeat/input/filestream/environment_test.go +++ b/filebeat/input/filestream/environment_test.go @@ -386,6 +386,7 @@ func getIDFromPath(filepath, inputID string, fi os.FileInfo) string { // waitUntilEventCount waits until total count events arrive to the client. func (e *inputTestingEnvironment) waitUntilEventCount(count int) { + e.t.Helper() msg := &strings.Builder{} require.Eventuallyf(e.t, func() bool { msg.Reset() diff --git a/filebeat/input/filestream/input_integration_test.go b/filebeat/input/filestream/input_integration_test.go index a8a950e377f4..5c063481dd53 100644 --- a/filebeat/input/filestream/input_integration_test.go +++ b/filebeat/input/filestream/input_integration_test.go @@ -55,9 +55,9 @@ func TestFilestreamCloseRenamed(t *testing.T) { "id": id, "paths": []string{env.abspath(testlogName) + "*"}, "prospector.scanner.check_interval": "10ms", - "prospector.scanner.fingerprint.enabled": false, "close.on_state_change.check_interval": "1ms", "close.on_state_change.renamed": "true", + "prospector.scanner.fingerprint.enabled": false, "file_identity.native": map[string]any{}, }) @@ -540,11 +540,13 @@ func TestFilestreamCloseAfterIntervalRotatedAndNewRemoved(t *testing.T) { testlogName := "test.log" id := "fake-ID-" + uuid.Must(uuid.NewV4()).String() inp := env.mustCreateInput(map[string]interface{}{ - "id": id, - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", - "close.on_state_change.check_interval": "10ms", - "close.on_state_change.inactive": "100ms", + "id": id, + "paths": []string{env.abspath(testlogName)}, + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": "false", + "prospector.scanner.check_interval": "1ms", + "close.on_state_change.check_interval": "10ms", + "close.on_state_change.inactive": "100ms", // reader is not stopped when file is removed to see if the reader can still detect // if the file has been inactive even if it have been removed in the meantime "close.on_state_change.removed": "false", @@ -707,10 +709,12 @@ func TestFilestreamTruncateBigScannerInterval(t *testing.T) { testlogName := "test.log" id := "fake-ID-" + uuid.Must(uuid.NewV4()).String() inp := env.mustCreateInput(map[string]interface{}{ - "id": id, - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "5s", - "prospector.scanner.resend_on_touch": "true", + "id": id, + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "5s", + "prospector.scanner.resend_on_touch": "true", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, }) ctx, cancelInput := context.WithCancel(context.Background()) diff --git a/filebeat/input/filestream/legacy_metrics_integration_test.go b/filebeat/input/filestream/legacy_metrics_integration_test.go index 649ede41f3e2..ec2e18a3706b 100644 --- a/filebeat/input/filestream/legacy_metrics_integration_test.go +++ b/filebeat/input/filestream/legacy_metrics_integration_test.go @@ -41,6 +41,8 @@ filebeat.inputs: enabled: true close.reader.after_interval: 1s prospector.scanner.check_interval: 500ms + file_identity.native: ~ + prospector.scanner.fingerprint.enabled: false paths: - %s/*.filestream - type: log @@ -48,6 +50,8 @@ filebeat.inputs: enabled: true close_timeout: 1s scan_frequency: 500ms + file_identity.native: ~ + prospector.scanner.fingerprint.enabled: false paths: - %s/*.log @@ -71,7 +75,9 @@ func TestLegacyMetrics(t *testing.T) { filebeat.WriteConfigFile(cfg) filebeat.Start() - filebeat.WaitForLogs("Metrics endpoint listening on:", 10*time.Second) + filebeat.WaitForLogs("Metrics endpoint listening on:", + 10*time.Second, + "metrics endpoint did not start") // After starting Filebeat all counters must be zero waitForMetrics(t, diff --git a/filebeat/input/filestream/metrics_integration_test.go b/filebeat/input/filestream/metrics_integration_test.go index 3671f076d0ed..b551b2321b76 100644 --- a/filebeat/input/filestream/metrics_integration_test.go +++ b/filebeat/input/filestream/metrics_integration_test.go @@ -33,11 +33,13 @@ func TestFilestreamMetrics(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "24h", - "close.on_state_change.check_interval": "100ms", - "close.on_state_change.inactive": "2s", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "24h", + "close.on_state_change.check_interval": "100ms", + "close.on_state_change.inactive": "2s", + "prospector.scanner.fingerprint.enabled": false, + "file_identity.native": map[string]any{}, }) testlines := []byte("first line\nsecond line\nthird line\n") diff --git a/filebeat/input/filestream/parsers_integration_test.go b/filebeat/input/filestream/parsers_integration_test.go index 619d39f05126..858f4e6d1ce3 100644 --- a/filebeat/input/filestream/parsers_integration_test.go +++ b/filebeat/input/filestream/parsers_integration_test.go @@ -29,9 +29,11 @@ func TestParsersAgentLogs(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "ndjson": map[string]interface{}{ @@ -65,9 +67,11 @@ func TestParsersIncludeMessage(t *testing.T) { testlogName := "test.log" readLine := "include this" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "100ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "100ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "include_message": map[string]interface{}{ @@ -98,9 +102,11 @@ func TestParsersDockerLogsFiltering(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "ndjson": map[string]interface{}{ @@ -137,9 +143,11 @@ func TestParsersSimpleJSONOverwrite(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "ndjson": map[string]interface{}{ @@ -173,9 +181,11 @@ func TestParsersTimestampInJSONMessage(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "ndjson": map[string]interface{}{ @@ -214,9 +224,11 @@ func TestParsersJavaElasticsearchLogs(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "multiline": map[string]interface{}{ @@ -249,9 +261,11 @@ func TestParsersCStyleLog(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "multiline": map[string]interface{}{ @@ -290,9 +304,11 @@ func TestParsersRabbitMQMultilineLog(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "multiline": map[string]interface{}{ @@ -335,9 +351,11 @@ func TestParsersMultilineMaxLines(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "multiline": map[string]interface{}{ @@ -379,9 +397,11 @@ func TestParsersMultilineTimeout(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "multiline": map[string]interface{}{ @@ -444,10 +464,12 @@ func TestParsersMultilineMaxBytes(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", - "message_max_bytes": 50, + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "message_max_bytes": 50, + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "multiline": map[string]interface{}{ @@ -486,10 +508,12 @@ func TestParsersCloseTimeoutWithMultiline(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", - "close.reader.after_interval": "1s", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "close.reader.after_interval": "1s", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "multiline": map[string]interface{}{ @@ -551,10 +575,12 @@ func TestParsersConsecutiveNewline(t *testing.T) { testlogName := "test.log" inp := env.mustCreateInput(map[string]interface{}{ - "id": "fake-ID", - "paths": []string{env.abspath(testlogName)}, - "prospector.scanner.check_interval": "1ms", - "close.reader.after_interval": "1s", + "id": "fake-ID", + "paths": []string{env.abspath(testlogName)}, + "prospector.scanner.check_interval": "1ms", + "close.reader.after_interval": "1s", + "file_identity.native": map[string]any{}, + "prospector.scanner.fingerprint.enabled": false, "parsers": []map[string]interface{}{ { "multiline": map[string]interface{}{