From e8d0bbba6c40b4113f429698f7182209561662e1 Mon Sep 17 00:00:00 2001 From: Florian Zipperle Date: Wed, 30 Aug 2023 22:21:57 +0200 Subject: [PATCH 1/3] Fix Stringer implementation of fingerprint processor --- CHANGELOG-developer.next.asciidoc | 1 + libbeat/processors/fingerprint/config.go | 25 ++++++++++++++---- libbeat/processors/fingerprint/encode.go | 20 ++++++++++---- libbeat/processors/fingerprint/fingerprint.go | 7 +++-- .../fingerprint/fingerprint_test.go | 13 ++++++++++ libbeat/processors/fingerprint/hash.go | 26 +++++++++++++------ 6 files changed, 70 insertions(+), 22 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 3b087d19a9d..7877984b5b5 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -86,6 +86,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757] - Renamed an httpjson input metric to follow naming conventions. `httpjson_interval_pages_total` was renamed to `httpjson_interval_pages` because the `_total` suffix is reserved for counters. {issue}35933[35933] {pull}36169[36169] - Fixed some race conditions in tests {pull}36185[36185] +- Fix Stringer implementation of fingerprint processor {issue}35174[35174] ==== Added diff --git a/libbeat/processors/fingerprint/config.go b/libbeat/processors/fingerprint/config.go index dc36b6bceff..2f31691e741 100644 --- a/libbeat/processors/fingerprint/config.go +++ b/libbeat/processors/fingerprint/config.go @@ -17,13 +17,15 @@ package fingerprint +import "encoding/json" + // Config for fingerprint processor. type Config struct { - Method hashMethod `config:"method"` // Hash function to use for fingerprinting - Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from - TargetField string `config:"target_field"` // Target field for the fingerprint - Encoding encodingMethod `config:"encoding"` // Encoding to use for target field value - IgnoreMissing bool `config:"ignore_missing"` // Ignore missing fields? + Method namedHashMethod `config:"method"` // Hash function to use for fingerprinting + Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from + TargetField string `config:"target_field"` // Target field for the fingerprint + Encoding namedEncodingMethod `config:"encoding"` // Encoding to use for target field value + IgnoreMissing bool `config:"ignore_missing"` // Ignore missing fields? } func defaultConfig() Config { @@ -34,3 +36,16 @@ func defaultConfig() Config { IgnoreMissing: false, } } + +func (c *Config) MarshalJSON() ([]byte, error) { + type Alias Config + return json.Marshal(&struct { + Method string + Encoding string + *Alias + }{ + Method: c.Method.Name, + Encoding: c.Encoding.Name, + Alias: (*Alias)(c), + }) +} diff --git a/libbeat/processors/fingerprint/encode.go b/libbeat/processors/fingerprint/encode.go index 843c7bd5d29..dd04068df73 100644 --- a/libbeat/processors/fingerprint/encode.go +++ b/libbeat/processors/fingerprint/encode.go @@ -24,16 +24,26 @@ import ( "strings" ) +type namedEncodingMethod struct { + Name string + Encode encodingMethod +} type encodingMethod func([]byte) string -var encodings = map[string]encodingMethod{ - "hex": hex.EncodeToString, - "base32": base32.StdEncoding.EncodeToString, - "base64": base64.StdEncoding.EncodeToString, +var encodings = map[string]namedEncodingMethod{} + +func init() { + for _, e := range []namedEncodingMethod{ + {Name: "hex", Encode: hex.EncodeToString}, + {Name: "base32", Encode: base32.StdEncoding.EncodeToString}, + {Name: "base64", Encode: base64.StdEncoding.EncodeToString}, + } { + encodings[e.Name] = e + } } // Unpack creates the encodingMethod from the given string -func (e *encodingMethod) Unpack(str string) error { +func (e *namedEncodingMethod) Unpack(str string) error { str = strings.ToLower(str) m, found := encodings[str] diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 3f22082bad4..fdbcf158b27 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -60,7 +60,7 @@ func New(cfg *config.C) (beat.Processor, error) { p := &fingerprint{ config: config, - hash: config.Method, + hash: config.Method.Hash, fields: fields, } @@ -75,7 +75,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { return nil, makeErrComputeFingerprint(err) } - encodedHash := p.config.Encoding(hashFn.Sum(nil)) + encodedHash := p.config.Encoding.Encode(hashFn.Sum(nil)) if _, err := event.PutValue(p.config.TargetField, encodedHash); err != nil { return nil, makeErrComputeFingerprint(err) @@ -85,8 +85,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { } func (p *fingerprint) String() string { - //nolint:staticcheck // https://github.com/elastic/beats/issues/35174 - json, _ := json.Marshal(p.config) + json, _ := json.Marshal(&p.config) return procName + "=" + string(json) } diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index ead0bc2c005..a988f163d3d 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -18,6 +18,7 @@ package fingerprint import ( + "fmt" "math/rand" "strconv" "testing" @@ -459,6 +460,18 @@ func TestIgnoreMissing(t *testing.T) { } } +func TestProcessorStringer(t *testing.T) { + testConfig, err := config.NewConfigFrom(mapstr.M{ + "fields": []string{"field1"}, + "encoding": "hex", + "method": "md5", + }) + assert.NoError(t, err) + p, err := New(testConfig) + assert.NoError(t, err) + assert.Equal(t, `fingerprint={"Method":"md5","Encoding":"hex","Fields":["field1"],"TargetField":"fingerprint","IgnoreMissing":false}`, fmt.Sprint(p)) +} + func BenchmarkHashMethods(b *testing.B) { events := nRandomEvents(100000) diff --git a/libbeat/processors/fingerprint/hash.go b/libbeat/processors/fingerprint/hash.go index 1c4af0d0161..1c8cf146a14 100644 --- a/libbeat/processors/fingerprint/hash.go +++ b/libbeat/processors/fingerprint/hash.go @@ -28,19 +28,29 @@ import ( "github.com/cespare/xxhash/v2" ) +type namedHashMethod struct { + Name string + Hash hashMethod +} type hashMethod func() hash.Hash -var hashes = map[string]hashMethod{ - "md5": md5.New, - "sha1": sha1.New, - "sha256": sha256.New, - "sha384": sha512.New384, - "sha512": sha512.New, - "xxhash": newXxHash, +var hashes = map[string]namedHashMethod{} + +func init() { + for _, h := range []namedHashMethod{ + {Name: "md5", Hash: md5.New}, + {Name: "sha1", Hash: sha1.New}, + {Name: "sha256", Hash: sha256.New}, + {Name: "sha384", Hash: sha512.New384}, + {Name: "sha512", Hash: sha512.New}, + {Name: "xxhash", Hash: newXxHash}, + } { + hashes[h.Name] = h + } } // Unpack creates the hashMethod from the given string -func (f *hashMethod) Unpack(str string) error { +func (f *namedHashMethod) Unpack(str string) error { str = strings.ToLower(str) m, found := hashes[str] From 5c6fb0514a2df208b85f652df895070ccc189198 Mon Sep 17 00:00:00 2001 From: Florian Zipperle Date: Sun, 5 Nov 2023 11:21:11 +0100 Subject: [PATCH 2/3] Fix the linter issues --- libbeat/processors/fingerprint/fingerprint_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index a988f163d3d..9e51f04b832 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -78,6 +78,7 @@ func TestWithConfig(t *testing.T) { Fields: test.input.Clone(), } newEvent, err := p.Run(testEvent) + assert.NoError(t, err) v, err := newEvent.GetValue("fingerprint") assert.NoError(t, err) assert.Equal(t, test.want, v) @@ -485,8 +486,8 @@ func BenchmarkHashMethods(b *testing.B) { b.Run(method, func(b *testing.B) { b.ResetTimer() - for _, e := range events { - _, err := p.Run(&e) + for i := range events { + _, err := p.Run(&events[i]) if err != nil { b.Fatal(err) } @@ -504,7 +505,7 @@ func nRandomEvents(num int) []beat.Event { charsetLen := len(charset) b := make([]byte, 200) - var events []beat.Event + events := make([]beat.Event, num) for i := 0; i < num; i++ { for j := range b { b[j] = charset[prng.Intn(charsetLen)] From b39c30c2b6e26fb7c38b0a475413c2fe5a195cb4 Mon Sep 17 00:00:00 2001 From: florian Date: Fri, 2 Feb 2024 10:13:19 +0100 Subject: [PATCH 3/3] Tests: use require instead of assert. --- libbeat/processors/fingerprint/fingerprint_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 9e51f04b832..5f6bdb70b5e 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -467,10 +467,10 @@ func TestProcessorStringer(t *testing.T) { "encoding": "hex", "method": "md5", }) - assert.NoError(t, err) + require.NoError(t, err) p, err := New(testConfig) - assert.NoError(t, err) - assert.Equal(t, `fingerprint={"Method":"md5","Encoding":"hex","Fields":["field1"],"TargetField":"fingerprint","IgnoreMissing":false}`, fmt.Sprint(p)) + require.NoError(t, err) + require.Equal(t, `fingerprint={"Method":"md5","Encoding":"hex","Fields":["field1"],"TargetField":"fingerprint","IgnoreMissing":false}`, fmt.Sprint(p)) } func BenchmarkHashMethods(b *testing.B) {