Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Stringer implementation of fingerprint processor #36468

Merged
merged 10 commits into from
Feb 5, 2024
1 change: 1 addition & 0 deletions CHANGELOG-developer.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
- Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757]
- Renamed an httpjson input metric to follow naming conventions. `httpjson_interval_pages_total` was renamed to `httpjson_interval_pages` because the `_total` suffix is reserved for counters. {issue}35933[35933] {pull}36169[36169]
- Fixed some race conditions in tests {pull}36185[36185]
- Fix Stringer implementation of fingerprint processor {issue}35174[35174]
- Re-enable HTTPJSON fixed flakey test. {issue}34929[34929] {pull}36525[36525]
- Make winlogbeat/sys/wineventlog follow the unsafe.Pointer rules. {pull}36650[36650]
- Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714]
Expand Down
25 changes: 20 additions & 5 deletions libbeat/processors/fingerprint/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@

package fingerprint

import "encoding/json"

// Config for fingerprint processor.
type Config struct {
Method hashMethod `config:"method"` // Hash function to use for fingerprinting
Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from
TargetField string `config:"target_field"` // Target field for the fingerprint
Encoding encodingMethod `config:"encoding"` // Encoding to use for target field value
IgnoreMissing bool `config:"ignore_missing"` // Ignore missing fields?
Method namedHashMethod `config:"method"` // Hash function to use for fingerprinting
Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from
TargetField string `config:"target_field"` // Target field for the fingerprint
Encoding namedEncodingMethod `config:"encoding"` // Encoding to use for target field value
IgnoreMissing bool `config:"ignore_missing"` // Ignore missing fields?
}

func defaultConfig() Config {
Expand All @@ -34,3 +36,16 @@ func defaultConfig() Config {
IgnoreMissing: false,
}
}

func (c *Config) MarshalJSON() ([]byte, error) {
type Alias Config
return json.Marshal(&struct {
Method string
Encoding string
*Alias
}{
Method: c.Method.Name,
Encoding: c.Encoding.Name,
Alias: (*Alias)(c),
})
}
20 changes: 15 additions & 5 deletions libbeat/processors/fingerprint/encode.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,26 @@ import (
"strings"
)

type namedEncodingMethod struct {
Name string
Encode encodingMethod
}
type encodingMethod func([]byte) string

var encodings = map[string]encodingMethod{
"hex": hex.EncodeToString,
"base32": base32.StdEncoding.EncodeToString,
"base64": base64.StdEncoding.EncodeToString,
var encodings = map[string]namedEncodingMethod{}

func init() {
for _, e := range []namedEncodingMethod{
{Name: "hex", Encode: hex.EncodeToString},
{Name: "base32", Encode: base32.StdEncoding.EncodeToString},
{Name: "base64", Encode: base64.StdEncoding.EncodeToString},
} {
encodings[e.Name] = e
}
}

// Unpack creates the encodingMethod from the given string
func (e *encodingMethod) Unpack(str string) error {
func (e *namedEncodingMethod) Unpack(str string) error {
str = strings.ToLower(str)

m, found := encodings[str]
Expand Down
7 changes: 3 additions & 4 deletions libbeat/processors/fingerprint/fingerprint.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func New(cfg *config.C) (beat.Processor, error) {

p := &fingerprint{
config: config,
hash: config.Method,
hash: config.Method.Hash,
fields: fields,
}

Expand All @@ -75,7 +75,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) {
return nil, makeErrComputeFingerprint(err)
}

encodedHash := p.config.Encoding(hashFn.Sum(nil))
encodedHash := p.config.Encoding.Encode(hashFn.Sum(nil))

if _, err := event.PutValue(p.config.TargetField, encodedHash); err != nil {
return nil, makeErrComputeFingerprint(err)
Expand All @@ -85,8 +85,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) {
}

func (p *fingerprint) String() string {
//nolint:staticcheck // https://github.com/elastic/beats/issues/35174
json, _ := json.Marshal(p.config)
json, _ := json.Marshal(&p.config)
return procName + "=" + string(json)
}

Expand Down
20 changes: 17 additions & 3 deletions libbeat/processors/fingerprint/fingerprint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package fingerprint

import (
"fmt"
"math/rand"
"strconv"
"testing"
Expand Down Expand Up @@ -77,6 +78,7 @@ func TestWithConfig(t *testing.T) {
Fields: test.input.Clone(),
}
newEvent, err := p.Run(testEvent)
assert.NoError(t, err)
v, err := newEvent.GetValue("fingerprint")
assert.NoError(t, err)
assert.Equal(t, test.want, v)
Expand Down Expand Up @@ -459,6 +461,18 @@ func TestIgnoreMissing(t *testing.T) {
}
}

func TestProcessorStringer(t *testing.T) {
testConfig, err := config.NewConfigFrom(mapstr.M{
"fields": []string{"field1"},
"encoding": "hex",
"method": "md5",
})
assert.NoError(t, err)
p, err := New(testConfig)
assert.NoError(t, err)
assert.Equal(t, `fingerprint={"Method":"md5","Encoding":"hex","Fields":["field1"],"TargetField":"fingerprint","IgnoreMissing":false}`, fmt.Sprint(p))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggest using require.* instead of assert.*.

}

func BenchmarkHashMethods(b *testing.B) {
events := nRandomEvents(100000)

Expand All @@ -472,8 +486,8 @@ func BenchmarkHashMethods(b *testing.B) {

b.Run(method, func(b *testing.B) {
b.ResetTimer()
for _, e := range events {
_, err := p.Run(&e)
for i := range events {
_, err := p.Run(&events[i])
if err != nil {
b.Fatal(err)
}
Expand All @@ -491,7 +505,7 @@ func nRandomEvents(num int) []beat.Event {
charsetLen := len(charset)
b := make([]byte, 200)

var events []beat.Event
events := make([]beat.Event, num)
for i := 0; i < num; i++ {
for j := range b {
b[j] = charset[prng.Intn(charsetLen)]
Expand Down
26 changes: 18 additions & 8 deletions libbeat/processors/fingerprint/hash.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,29 @@ import (
"github.com/cespare/xxhash/v2"
)

type namedHashMethod struct {
Name string
Hash hashMethod
}
type hashMethod func() hash.Hash

var hashes = map[string]hashMethod{
"md5": md5.New,
"sha1": sha1.New,
"sha256": sha256.New,
"sha384": sha512.New384,
"sha512": sha512.New,
"xxhash": newXxHash,
var hashes = map[string]namedHashMethod{}

func init() {
for _, h := range []namedHashMethod{
{Name: "md5", Hash: md5.New},
{Name: "sha1", Hash: sha1.New},
{Name: "sha256", Hash: sha256.New},
{Name: "sha384", Hash: sha512.New384},
{Name: "sha512", Hash: sha512.New},
{Name: "xxhash", Hash: newXxHash},
} {
hashes[h.Name] = h
}
}

// Unpack creates the hashMethod from the given string
func (f *hashMethod) Unpack(str string) error {
func (f *namedHashMethod) Unpack(str string) error {
str = strings.ToLower(str)

m, found := hashes[str]
Expand Down
Loading