diff --git a/.chloggen/32486-apply-semantic-conventions.yaml b/.chloggen/32486-apply-semantic-conventions.yaml new file mode 100644 index 000000000000..3cdf0e1fce40 --- /dev/null +++ b/.chloggen/32486-apply-semantic-conventions.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: azureeventhubreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Map Azure Resource Log property names to the Semantic Conventions equivalent" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [32764] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: '[user]' \ No newline at end of file diff --git a/pkg/translator/azure_logs/Makefile b/pkg/translator/azure_logs/Makefile new file mode 100644 index 000000000000..bdd863a203be --- /dev/null +++ b/pkg/translator/azure_logs/Makefile @@ -0,0 +1 @@ +include ../../../Makefile.Common diff --git a/pkg/translator/azure_logs/complex_conversions.go b/pkg/translator/azure_logs/complex_conversions.go new file mode 100644 index 000000000000..48875ff269fa --- /dev/null +++ b/pkg/translator/azure_logs/complex_conversions.go @@ -0,0 +1,102 @@ +package azure_logs + +import ( + "fmt" + "strconv" + "strings" +) + +type ComplexConversion func(string, any, map[string]any) bool +type TypeConversion func(string, any, map[string]any, string) bool + +var conversions = map[string]ComplexConversion{ + "AzureCdnAccessLog:SecurityProtocol": azureCdnAccessLogSecurityProtocol, + "FrontDoorAccessLog:securityProtocol": azureCdnAccessLogSecurityProtocol, + "AppServiceHTTPLogs:Protocol": appServiceHTTPLogsProtocol, + "AppServiceHTTPLogs:TimeTaken": appServiceHTTPLogTimeTakenMilliseconds, + "FrontDoorHealthProbeLog:DNSLatencyMicroseconds": frontDoorHealthProbeLogDNSLatencyMicroseconds, + "FrontDoorHealthProbeLog:totalLatencyMilliseconds": frontDoorHealthProbeLogTotalLatencyMilliseconds, +} + +// Splits the "TLS 1.2" value into "TLS" and "1.2" and sets as "network.protocol.name" and "network.protocol.version" +func azureCdnAccessLogSecurityProtocol(key string, value any, attrs map[string]any) bool { + if str, ok := value.(string); ok { + if parts := strings.SplitN(str, " ", 2); len(parts) == 2 { + attrs["tls.protocol.name"] = strings.ToLower(parts[0]) + attrs["tls.protocol.version"] = parts[1] + return true + } + } + return false +} + +// Splits the "HTTP/1.1" value into "HTTP" and "1.1" and sets as "network.protocol.name" and "network.protocol.version" +func appServiceHTTPLogsProtocol(key string, value any, attrs map[string]any) bool { + if str, ok := value.(string); ok { + if parts := strings.SplitN(str, "/", 2); len(parts) == 2 { + attrs["network.protocol.name"] = strings.ToLower(parts[0]) + attrs["network.protocol.version"] = parts[1] + return true + } + } + return false +} + +// Converts Microseconds value to Seconds and sets as "dns.lookup.duration" +func frontDoorHealthProbeLogDNSLatencyMicroseconds(key string, value any, attrs map[string]any) bool { + microseconds, ok := tryParseFloat64(value) + if !ok { + return false + } + seconds := microseconds / 1_000_000 + attrs["dns.lookup.duration"] = seconds + return true +} + +// Converts Milliseconds value to Seconds and sets as "http.client.request.duration" +func frontDoorHealthProbeLogTotalLatencyMilliseconds(key string, value any, attrs map[string]any) bool { + milliseconds, ok := tryParseFloat64(value) + if !ok { + return false + } + seconds := milliseconds / 1_000 + attrs["http.request.duration"] = seconds + return true +} + +// Converts Milliseconds value to Seconds and sets as "http.server.request.duration" +func appServiceHTTPLogTimeTakenMilliseconds(key string, value any, attrs map[string]any) bool { + milliseconds, ok := tryParseFloat64(value) + if !ok { + return false + } + seconds := milliseconds / 1_000 + attrs["http.server.request.duration"] = seconds + return true +} + +func tryParseFloat64(value any) (float64, bool) { + switch value.(type) { + case float32: + return float64(value.(float32)), true + case float64: + return value.(float64), true + case int: + return float64(value.(int)), true + case int32: + return float64(value.(int32)), true + case int64: + return float64(value.(int64)), true + case string: + f, err := strconv.ParseFloat(value.(string), 64) + return f, err == nil + default: + return 0, false + } +} + +func tryGetComplexConversion(category string, propertyName string) (ComplexConversion, bool) { + key := fmt.Sprintf("%s:%s", category, propertyName) + conversion, ok := conversions[key] + return conversion, ok +} diff --git a/pkg/translator/azure_logs/complex_conversions_test.go b/pkg/translator/azure_logs/complex_conversions_test.go new file mode 100644 index 000000000000..c8963fd5df4b --- /dev/null +++ b/pkg/translator/azure_logs/complex_conversions_test.go @@ -0,0 +1,72 @@ +package azure_logs + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestFrontDoorAccessLogSecurityProtocol(t *testing.T) { + f, ok := tryGetComplexConversion("FrontDoorAccessLog", "securityProtocol") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("securityProtocol", "TLS 1.2", attrs) + assert.True(t, ok) + protocolName, ok := attrs["tls.protocol.name"] + assert.True(t, ok) + // Protocol name is normalized to lower case + assert.Equal(t, "tls", protocolName) + protocolVersion, ok := attrs["tls.protocol.version"] + assert.True(t, ok) + assert.Equal(t, "1.2", protocolVersion) +} + +func TestAzureCDNAccessLogSecurityProtocol(t *testing.T) { + f, ok := tryGetComplexConversion("AzureCDNAccessLog", "SecurityProtocol") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("SecurityProtocol", "TLS 1.2", attrs) + assert.True(t, ok) + protocolName, ok := attrs["tls.protocol.name"] + assert.True(t, ok) + // Protocol name is normalized to lower case + assert.Equal(t, "tls", protocolName) + protocolVersion, ok := attrs["tls.protocol.version"] + assert.True(t, ok) + assert.Equal(t, "1.2", protocolVersion) +} + +func TestAppServiceHTTPLogsProtocol(t *testing.T) { + f, ok := tryGetComplexConversion("AppServiceHTTPLogs", "Protocol") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("Protocol", "HTTP/1.1", attrs) + assert.True(t, ok) + protocolName, ok := attrs["network.protocol.name"] + assert.True(t, ok) + assert.Equal(t, "http", protocolName) + protocolVersion, ok := attrs["network.protocol.version"] + assert.True(t, ok) + assert.Equal(t, "1.1", protocolVersion) +} + +func TestFrontDoorHealthProbeLogDNSLatencyMicroseconds(t *testing.T) { + f, ok := tryGetComplexConversion("FrontDoorHealthProbeLog", "DNSLatencyMicroseconds") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("DNSLatencyMicroseconds", 123456, attrs) + assert.True(t, ok) + duration, ok := attrs["dns.lookup.duration"].(float64) + assert.True(t, ok) + assert.Equal(t, 0.123456, duration) +} + +func TestFrontDoorHealthProbeLogTotalLatencyMilliseconds(t *testing.T) { + f, ok := tryGetComplexConversion("FrontDoorHealthProbeLog", "totalLatencyMilliseconds") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("totalLatencyMilliseconds", 123, attrs) + assert.True(t, ok) + duration, ok := attrs["http.request.duration"].(float64) + assert.True(t, ok) + assert.Equal(t, 0.123, duration) +} diff --git a/pkg/translator/azure_logs/go.mod b/pkg/translator/azure_logs/go.mod new file mode 100644 index 000000000000..3709bc5bda0b --- /dev/null +++ b/pkg/translator/azure_logs/go.mod @@ -0,0 +1,53 @@ +//module github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs + +go 1.21.0 + +require ( + github.com/json-iterator/go v1.1.12 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.99.0 + github.com/relvacode/iso8601 v1.4.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/component v0.99.1-0.20240503221155-67d37183e6ac + go.opentelemetry.io/collector/pdata v1.6.1-0.20240503221155-67d37183e6ac + go.opentelemetry.io/collector/semconv v0.99.1-0.20240503221155-67d37183e6ac + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.99.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.99.1-0.20240503221155-67d37183e6ac // indirect + go.opentelemetry.io/collector/confmap v0.99.1-0.20240503221155-67d37183e6ac // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pdatatest + +module github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../golden diff --git a/pkg/translator/azure_logs/go.sum b/pkg/translator/azure_logs/go.sum new file mode 100644 index 000000000000..a84ac60f8d4e --- /dev/null +++ b/pkg/translator/azure_logs/go.sum @@ -0,0 +1,85 @@ +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/relvacode/iso8601 v1.4.0 h1:GsInVSEJfkYuirYFxa80nMLbH2aydgZpIf52gYZXUJs= +github.com/relvacode/iso8601 v1.4.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/component v0.99.1-0.20240503221155-67d37183e6ac/go.mod h1:+b56nMIvo3CO5TShFn38RwX4FsXv0lVt2HoGmsaXObo= +go.opentelemetry.io/collector/config/configtelemetry v0.99.1-0.20240503221155-67d37183e6ac/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= +go.opentelemetry.io/collector/confmap v0.99.1-0.20240503221155-67d37183e6ac/go.mod h1:BWKPIpYeUzSG6ZgCJMjF7xsLvyrvJCfYURl57E5vhiQ= +go.opentelemetry.io/collector/pdata v1.6.1-0.20240503221155-67d37183e6ac h1:+FnNEftMuQPg86UOZnLUXzdIjxmHKNsnmSiRTYTCVok= +go.opentelemetry.io/collector/pdata v1.6.1-0.20240503221155-67d37183e6ac/go.mod h1:ehCBBA5GoFrMZkwyZAKGY/lAVSgZf6rzUt3p9mddmPU= +go.opentelemetry.io/collector/semconv v0.99.1-0.20240503221155-67d37183e6ac h1:FGz+i1cQrlahOmW3XAcM372XeNJPk57FYDphGVPaFwU= +go.opentelemetry.io/collector/semconv v0.99.1-0.20240503221155-67d37183e6ac/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/translator/azure_logs/metadata.yaml b/pkg/translator/azure_logs/metadata.yaml new file mode 100644 index 000000000000..1fb3dbfbcb4d --- /dev/null +++ b/pkg/translator/azure_logs/metadata.yaml @@ -0,0 +1,3 @@ +status: + codeowners: + active: [open-telemetry/collector-approvers, atoulme, cparkins] diff --git a/pkg/translator/azure_logs/normalize.go b/pkg/translator/azure_logs/normalize.go new file mode 100644 index 000000000000..06c47bb76363 --- /dev/null +++ b/pkg/translator/azure_logs/normalize.go @@ -0,0 +1,76 @@ +package azure_logs + +import ( + "fmt" + "strconv" + "strings" +) + +const maxInt32 = int64(int32(^uint32(0) >> 1)) + +type valueNormalizer func(any) any + +var normalizers = map[string]valueNormalizer{ + "http.request.body.size": toInt, + "http.request.size": toInt, + "http.response.body.size": toInt, + "http.response.size": toInt, + "http.response.status_code": toInt, + "http.server.request.duration": toFloat, + "network.protocol.name": toLower, + "server.port": toInt, +} + +func normalizeValue(key string, val any) any { + if f, exists := normalizers[key]; exists { + return f(val) + } + return val +} + +func toLower(value any) any { + switch value.(type) { + case string: + return strings.ToLower(value.(string)) + default: + return strings.ToLower(fmt.Sprint(value)) + } +} + +func toFloat(value any) any { + switch value.(type) { + case float32: + return float64(value.(float32)) + case float64: + return value.(float64) + case int: + return float64(value.(int)) + case int32: + return float64(value.(int32)) + case int64: + return float64(value.(int64)) + case string: + f, err := strconv.ParseFloat(value.(string), 64) + if err == nil { + return f + } + } + return value +} + +func toInt(value any) any { + switch value.(type) { + case int: + return int64(value.(int)) + case int32: + return int64(int(value.(int32))) + case int64: + return value.(int64) + case string: + i, err := strconv.ParseInt(value.(string), 10, 64) + if err == nil { + return i + } + } + return value +} diff --git a/pkg/translator/azure_logs/package_test.go b/pkg/translator/azure_logs/package_test.go new file mode 100644 index 000000000000..c4b240f8c62f --- /dev/null +++ b/pkg/translator/azure_logs/package_test.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package azure_logs + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/pkg/translator/azure_logs/property_names.go b/pkg/translator/azure_logs/property_names.go new file mode 100644 index 000000000000..f3067b6716f0 --- /dev/null +++ b/pkg/translator/azure_logs/property_names.go @@ -0,0 +1,295 @@ +package azure_logs + +var mappings = map[string]map[string]string{ + "common": {}, + "AzureCdnAccessLog": { + "BackendHostname": "destination.address", // If the request is being forwarded to a backend, this field represents the hostname of the backend. This field is blank if the request gets redirected or forwarded to a regional cache (when caching gets enabled for the routing rule). + "CacheStatus": "", // For caching scenarios, this field defines the cache hit/miss at the POP + "ClientIp": "client.address", // The IP address of the client that made the request. If there was an X-Forwarded-For header in the request, then the Client IP is picked from the same. + "ClientPort": "client.port", // The IP port of the client that made the request. + "HttpMethod": "http.request.method", // HTTP method used by the request. + "HttpStatusCode": "http.response.status_code", // The HTTP status code returned from the proxy. If a request to the origin timeouts the value for HttpStatusCode is set to 0. + "HttpStatusDetails": "", // Resulting status on the request. Meaning of this string value can be found at a Status reference table. + "HttpVersion": "network.protocol.version", // Type of the request or connection. + "POP": "", // Short name of the edge where the request landed. + "RequestBytes": "http.request.size", // The size of the HTTP request message in bytes, including the request headers and the request body. + "RequestUri": "url.full", // URI of the received request. + "ResponseBytes": "http.response.size", // Bytes sent by the backend server as the response. + "RoutingRuleName": "", // The name of the routing rule that the request matched. + "RulesEngineMatchNames": "", // The names of the rules that the request matched. + "SecurityProtocol": "", // handled by complex_conversions + "isReceivedFromClient": "", // If true, it means that the request came from the client. If false, the request is a miss in the edge (child POP) and is responded from origin shield (parent POP). + "TimeTaken": "", // The length of time from first byte of request into Azure Front Door to last byte of response out, in seconds. + "TrackingReference": "az.service_request_id", // The unique reference string that identifies a request served by Azure Front Door, also sent as X-Azure-Ref header to the client. Required for searching details in the access logs for a specific request. + "UserAgent": "user_agent.original", // The browser type that the client used. + "ErrorInfo": "error.type", // This field contains the specific type of error to narrow down troubleshooting area. + "TimeToFirstByte": "", // The length of time in milliseconds from when Microsoft CDN receives the request to the time the first byte gets sent to the client. The time is measured only from the Microsoft side. Client-side data isn't measured. + "Result": "", // SSLMismatchedSNI is a status code that signifies a successful request with a mismatch warning between the Server Name Indication (SNI) and the host header. This status code implies domain fronting, a technique that violates Azure Front Door's terms of service. Requests with SSLMismatchedSNI will be rejected after January 22, 2024. + "SNI": "", // This field specifies the Server Name Indication (SNI) that is sent during the TLS/SSL handshake. It can be used to identify the exact SNI value if there was a SSLMismatchedSNI status code. Additionally, it can be compared with the host value in the requestUri field to detect and resolve the mismatch issue. + }, + "FrontDoorAccessLog": { + "trackingReference": "az.service_request_id", + "httpMethod": "http.request.method", + "httpVersion": "network.protocol.version", + "requestUri": "url.full", + "hostName": "server.address", + "requestBytes": "http.request.size", + "responseBytes": "http.response.size", + "userAgent": "user_agent.original", + "clientIp": "client.address", + "clientPort": "client.port", + "socketIp": "network.peer.address", + "timeTaken": "http.server.request.duration", + "requestProtocol": "network.protocol.name", + "securityProtocol": "", // handled by complex_conversions + "securityCipher": "tls.cipher", + "securityCurves": "tls.curve", + "endpoint": "", + "httpStatusCode": "http.response.status_code", + "pop": "", + "cacheStatus": "", + "matchedRulesSetName": "", + "routeName": "http.route", + "referer": "http.request.header.referer", + "timeToFirstByte": "", + "errorInfo": "error.type", + "originURL": "", + "originIP": "", + "originName": "", + "result": "", + "sni": "", + }, + "FrontDoorHealthProbeLog": { + "healthProbeId": "", + "POP": "", + "httpVerb": "http.request.method", + "result": "", + "httpStatusCode": "http.response.status_code", + "probeURL": "url.full", + "originName": "", + "originIP": "server.address", + "totalLatencyMilliseconds": "", // handled by complex_conversions + "connectionLatencyMilliseconds": "", + "DNSLatencyMicroseconds": "", // handled by complex_conversions + }, + "FrontdoorWebApplicationFirewallLog": { + "clientIP": "client.address", + "clientPort": "client.port", + "socketIP": "network.peer.address", + "requestUri": "url.full", + "ruleName": "", + "policy": "", + "action": "", + "host": "server.address", + "trackingReference": "az.service_request_id", + "policyMode": "", + }, + "AppServiceAppLogs": { + "_BilledSize": "", //real The record size in bytes + "Category": "", //string Log category name + "ContainerId": "container.id", //string Application container id + "CustomLevel": "", //string Verbosity level of log + "ExceptionClass": "exception.type", //string Application class from where log message is emitted + "Host": "host.id", //string Host where the application is running + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", //string Verbosity level of log mapped to standard levels (Informational, Warning, Error, or Critical) + "Logger": "", //string Application logger used to emit log message + "Message": "", //string Log message + "Method": "code.function", //string Application Method from where log message is emitted + "OperationName": "", //string The name of the operation represented by this event. + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "ResultDescription": "", //string Log message description + "Source": "code.filepath", //string Application source from where log message is emitted + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "Stacktrace": "exception.stacktrace", //string Complete stack trace of the log message in case of exception + "StackTrace": "exception.stacktrace", //string Complete stack trace of the log message in case of exception + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "Type": "", //string The name of the table + "WebSiteInstanceId": "", //string Instance ID of the application running + }, + "AppServiceAuditLogs": { + "_BilledSize": "", //real The record size in bytes + "Category": "", //string Log category name + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "OperationName": "", //string Name of the operation + "Protocol": "network.protocol.name", //string Authentication protocol + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "Type": "", //string The name of the table + "User": "enduser.id", //string Username used for publishing access + "UserAddress": "client.address", //string Client IP address of the publishing user + "UserDisplayName": "", //string Email address of a user in case publishing was authorized via AAD authentication + }, + "AppServiceAuthenticationLogs": { + "_BilledSize": "", //real The record size in bytes + "CorrelationId": "", //string The ID for correlated events. + "Details": "", //string The event details. + "HostName": "", //string The host name of the application. + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", //string The level of log verbosity. + "Message": "", //string The log message. + "ModuleRuntimeVersion": "", //string The version of App Service Authentication running. + "OperationName": "", //string The name of the operation represented by this event. + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "SiteName": "", //string The runtime name of the application. + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "StatusCode": "http.response.status_code", //int The HTTP status code of the operation. + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "SubStatusCode": "", //int The HTTP sub-status code of the request. + "TaskName": "", //string The name of the task being performed. + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime The timestamp (UTC) of when this event was generated. + "Type": "", //string The name of the table + }, + "AppServiceConsoleLogs": { + "_BilledSize": "", // real The record size in bytes + "Category": "", // string Log category name + "ContainerId": "container.id", // string Application container id + "Host": "host.id", // string Host where the application is running + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", // string Verbosity level of log + "OperationName": "", // string The name of the operation represented by this event. + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "ResultDescription": "", // string Log message description + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time when event is generated + "Type": "", // string The name of the table + }, + "AppServiceEnvironmentPlatformLogs": { + "_BilledSize": "", // real The record size in bytes + "Category": "", // string + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "OperationName": "", // string + "ResourceId": "", // string + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "ResultDescription": "", // string + "ResultType": "", // string + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TimeGenerated": "", // datetime + "Type": "", // string The name of the table + }, + "AppServiceFileAuditLogs": { + "_BilledSize": "", // real The record size in bytes + "Category": "", // string Log category name + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "OperationName": "", // string Operation performed on a file + "Path": "", // string Path to the file that was changed + "Process": "", // string Type of the process that change the file + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time when event is generated + "Type": "", // string The name of the table + }, + "AppServiceHTTPLogs": { + "_BilledSize": "", //real The record size in bytes + "CIp": "client.address", //string IP address of the client + "ComputerName": "host.name", //string The name of the server on which the log file entry was generated. + "Cookie": "", //string Cookie on HTTP request + "CsBytes": "http.request.body.size", //int Number of bytes received by server + "CsHost": "url.domain", //string Host name header on HTTP request + "CsMethod": "http.request.method", //string The request HTTP verb + "CsUriQuery": "url.query", //string URI query on HTTP request + "CsUriStem": "url.path", //string The target of the request + "CsUsername": "", //string The name of the authenticated user on HTTP request + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Protocol": "", // handled by complex_conversions + "Referer": "http.request.header.referer", //string The site that the user last visited. This site provided a link to the current site + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "Result": "", //string Success / Failure of HTTP request + "ScBytes": "http.response.body.size", //int Number of bytes sent by server + "ScStatus": "http.response.status_code", //int HTTP status code + "ScSubStatus": "", //string Sub-status error code on HTTP request + "ScWin32Status": "", //string Windows status code on HTTP request + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "SPort": "server.port", //string Server port number + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "TimeTaken": "http.server.request.duration", //int Time taken by HTTP request in milliseconds + "Type": "", //string The name of the table + "UserAgent": "user_agent.original", //string User agent on HTTP request + }, + "AppServiceIPSecAuditLogs": { + "_BilledSize": "", // real The record size in bytes + "CIp": "client.address", // string IP address of the client + "CsHost": "url.domain", // string Host header of the HTTP request + "Details": "", // string Additional information + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "Result": "", // string The result whether the access is Allowed or Denied + "ServiceEndpoint": "", // string This indicates whether the access is via Virtual Network Service Endpoint communication + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time of the Http Request + "Type": "", // string The name of the table + "XAzureFDID": "http.request.header.x-azure-fdid", // string X-Azure-FDID header (Azure Frontdoor ID) of the HTTP request + "XFDHealthProbe": "http.request.header.x-fd-healthprobe", // string X-FD-HealthProbe (Azure Frontdoor Health Probe) of the HTTP request + "XForwardedFor": "http.request.header.x-forwarded-for", // string X-Forwarded-For header of the HTTP request + "XForwardedHost": "http.request.header.x-forwarded-host", // string X-Forwarded-Host header of the HTTP request + }, + "AppServicePlatformLogs": { + "ActivityId": "", // string Activity ID to correlate events + "_BilledSize": "", // real The record size in bytes + "containerId": "container.id", // string Application container id + "containerName": "container.name", // string Application container id + "DeploymentId": "", // string Deployment ID of the application deployment + "exception": "error.type", // string Details of the exception + "Host": "", // string Host where the application is running + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", // string Level of log verbosity + "Message": "", // string Log message + "OperationName": "", // string The name of the operation represented by this event. + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "StackTrace": "", // string Stack trace for the exception + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time when event is generated + "Type": "", // string The name of the table + }, + "AppServiceServerlessSecurityPluginData": { + "_BilledSize": "", // real The record size in bytes + "Index": "", // int Available when multiple payloads exist for the same message. In that case, payloads share the same SlSecRequestId and Index defines the chronological order of payloads. + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "MsgVersion": "", // string The version of the message schema. Used to make code changes backward- and forward- compatible. + "Payload": "", // dynamic An array of messages, where each one is a JSON string. + "PayloadType": "", // string The type of the payload. Mostly used to distinguish between messages meant for different types of security analysis. + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "Sender": "", // string The name of the component that published this message. Almost always will be the name of the plugin, but can also be platform. + "SlSecMetadata": "", // dynamic Contains details about the resource like the deployment ID, runtime info, website info, OS, etc. + "SlSecProps": "", // dynamic Contains other details that might be needed for debugging end-to-end requests, e.g., slsec nuget version. + "SlSecRequestId": "", // string The ingestion request ID used for identifying the message and the request for diagnostics and debugging. + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime The date and time (UTC) this message was created on the node. + "Type": "", // string The name of the table + }, +} + +func resourceLogKeyToSemConvKey(azName string, category string) (string, bool) { + mapping, ok := mappings[category] + if ok { + if mapped := mapping[azName]; mapped != "" { + return mapped, true + } + } + + mapping = mappings["common"] + if name := mapping[azName]; name != "" { + return name, true + } + + return "", false +} diff --git a/pkg/translator/azure_logs/resourcelogs_to_logs.go b/pkg/translator/azure_logs/resourcelogs_to_logs.go new file mode 100644 index 000000000000..c24a145cf86f --- /dev/null +++ b/pkg/translator/azure_logs/resourcelogs_to_logs.go @@ -0,0 +1,241 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package azure_logs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs" + +import ( + "bytes" + "encoding/json" + "errors" + "strconv" + + jsoniter "github.com/json-iterator/go" + "github.com/relvacode/iso8601" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + conventions "go.opentelemetry.io/collector/semconv/v1.13.0" + "go.uber.org/zap" + "golang.org/x/exp/slices" +) + +const ( + // Constants for OpenTelemetry Specs + scopeName = "otelcol/azureresourcelogs" + + // Constants for Azure Log Records + azureCategory = "azure.category" + azureCorrelationID = "azure.correlation.id" + azureDuration = "azure.duration" + azureIdentity = "azure.identity" + azureOperationName = "azure.operation.name" + azureOperationVersion = "azure.operation.version" + azureProperties = "azure.properties" + azureResourceID = "azure.resource.id" + azureResultType = "azure.result.type" + azureResultSignature = "azure.result.signature" + azureResultDescription = "azure.result.description" + azureTenantID = "azure.tenant.id" +) + +var ( + errMissingTimestamp = errors.New("missing timestamp") +) + +// azureRecords represents an array of Azure log records +// as exported via an Azure Event Hub +type azureRecords struct { + Records []azureLogRecord `json:"records"` +} + +// azureLogRecord represents a single Azure log following +// the common schema: +// https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/resource-logs-schema +type azureLogRecord struct { + Time string `json:"time"` + Timestamp string `json:"timeStamp"` + ResourceID string `json:"resourceId"` + TenantID *string `json:"tenantId"` + OperationName string `json:"operationName"` + OperationVersion *string `json:"operationVersion"` + Category string `json:"category"` + ResultType *string `json:"resultType"` + ResultSignature *string `json:"resultSignature"` + ResultDescription *string `json:"resultDescription"` + DurationMs *json.Number `json:"durationMs"` + CallerIPAddress *string `json:"callerIpAddress"` + CorrelationID *string `json:"correlationId"` + Identity *any `json:"identity"` + Level *json.Number `json:"Level"` + Location *string `json:"location"` + Properties *any `json:"properties"` +} + +var _ plog.Unmarshaler = (*ResourceLogsUnmarshaler)(nil) + +type ResourceLogsUnmarshaler struct { + Version string + Logger *zap.Logger +} + +func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { + l := plog.NewLogs() + + var azureLogs azureRecords + decoder := jsoniter.NewDecoder(bytes.NewReader(buf)) + if err := decoder.Decode(&azureLogs); err != nil { + return l, err + } + + var resourceIDs []string + azureResourceLogs := make(map[string][]azureLogRecord) + for _, azureLog := range azureLogs.Records { + azureResourceLogs[azureLog.ResourceID] = append(azureResourceLogs[azureLog.ResourceID], azureLog) + keyExists := slices.Contains(resourceIDs, azureLog.ResourceID) + if !keyExists { + resourceIDs = append(resourceIDs, azureLog.ResourceID) + } + } + + for _, resourceID := range resourceIDs { + logs := azureResourceLogs[resourceID] + resourceLogs := l.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, resourceID) + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName(scopeName) + scopeLogs.Scope().SetVersion(r.Version) + logRecords := scopeLogs.LogRecords() + + for i := 0; i < len(logs); i++ { + log := logs[i] + nanos, err := getTimestamp(log) + if err != nil { + r.Logger.Warn("Unable to convert timestamp from log", zap.String("timestamp", log.Time)) + continue + } + + lr := logRecords.AppendEmpty() + lr.SetTimestamp(nanos) + + if log.Level != nil { + severity := asSeverity(*log.Level) + lr.SetSeverityNumber(severity) + lr.SetSeverityText(log.Level.String()) + } + + if err := lr.Attributes().FromRaw(extractRawAttributes(log)); err != nil { + return l, err + } + } + } + + return l, nil +} + +func getTimestamp(record azureLogRecord) (pcommon.Timestamp, error) { + if record.Time != "" { + return asTimestamp(record.Time) + } else if record.Timestamp != "" { + return asTimestamp(record.Timestamp) + } + + return 0, errMissingTimestamp +} + +// asTimestamp will parse an ISO8601 string into an OpenTelemetry +// nanosecond timestamp. If the string cannot be parsed, it will +// return zero and the error. +func asTimestamp(s string) (pcommon.Timestamp, error) { + t, err := iso8601.ParseString(s) + if err != nil { + return 0, err + } + + return pcommon.Timestamp(t.UnixNano()), nil +} + +// asSeverity converts the Azure log level to equivalent +// OpenTelemetry severity numbers. If the log level is not +// valid, then the 'Unspecified' value is returned. +func asSeverity(number json.Number) plog.SeverityNumber { + switch number.String() { + case "Informational": + return plog.SeverityNumberInfo + case "Warning": + return plog.SeverityNumberWarn + case "Error": + return plog.SeverityNumberError + case "Critical": + return plog.SeverityNumberFatal + default: + var levelNumber, _ = number.Int64() + if levelNumber > 0 { + return plog.SeverityNumber(levelNumber) + } + + return plog.SeverityNumberUnspecified + } +} + +func extractRawAttributes(log azureLogRecord) map[string]any { + var attrs = map[string]any{} + + attrs[azureCategory] = log.Category + setIf(attrs, azureCorrelationID, log.CorrelationID) + if log.DurationMs != nil { + duration, err := strconv.ParseInt(log.DurationMs.String(), 10, 64) + if err == nil { + attrs[azureDuration] = duration + } + } + if log.Identity != nil { + attrs[azureIdentity] = *log.Identity + } + attrs[azureOperationName] = log.OperationName + setIf(attrs, azureOperationVersion, log.OperationVersion) + + if log.Properties != nil { + copyPropertiesAndApplySemanticConventions(log.Category, log.Properties, attrs) + } + + setIf(attrs, azureResultDescription, log.ResultDescription) + setIf(attrs, azureResultSignature, log.ResultSignature) + setIf(attrs, azureResultType, log.ResultType) + setIf(attrs, azureTenantID, log.TenantID) + + setIf(attrs, conventions.AttributeCloudRegion, log.Location) + attrs[conventions.AttributeCloudProvider] = conventions.AttributeCloudProviderAzure + + setIf(attrs, conventions.AttributeNetSockPeerAddr, log.CallerIPAddress) + return attrs +} + +func copyPropertiesAndApplySemanticConventions(category string, properties *any, attrs map[string]any) { + + pmap := (*properties).(map[string]any) + attrsProps := map[string]any{} + + for k, v := range pmap { + // Check for a complex conversion, e.g. AppServiceHTTPLogs.Protocol + if complexConversion, ok := tryGetComplexConversion(category, k); ok { + if complexConversion(k, v, attrs) { + continue + } + } + // Check for an equivalent Semantic Convention key + if otelKey, ok := resourceLogKeyToSemConvKey(k, category); ok { + attrs[otelKey] = normalizeValue(otelKey, v) + } else { + attrsProps[k] = v + } + } + + if len(attrsProps) > 0 { + attrs[azureProperties] = attrsProps + } +} + +func setIf(attrs map[string]any, key string, value *string) { + if value != nil && *value != "" { + attrs[key] = *value + } +} diff --git a/pkg/translator/azure_logs/resourcelogs_to_logs_test.go b/pkg/translator/azure_logs/resourcelogs_to_logs_test.go new file mode 100644 index 000000000000..124c3506d06d --- /dev/null +++ b/pkg/translator/azure_logs/resourcelogs_to_logs_test.go @@ -0,0 +1,658 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package azure_logs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs" + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + conventions "go.opentelemetry.io/collector/semconv/v1.13.0" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/plogtest" +) + +var testBuildInfo = component.BuildInfo{ + Version: "1.2.3", +} + +var minimumLogRecord = func() plog.LogRecord { + lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z") + lr.SetTimestamp(ts) + lr.Attributes().PutStr(azureOperationName, "SecretGet") + lr.Attributes().PutStr(azureCategory, "AuditEvent") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + return lr +}() + +var maximumLogRecord1 = func() plog.LogRecord { + lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z") + lr.SetTimestamp(ts) + lr.SetSeverityNumber(plog.SeverityNumberWarn) + lr.SetSeverityText("Warning") + guid := "607964b6-41a5-4e24-a5db-db7aab3b9b34" + + lr.Attributes().PutStr(azureTenantID, "/TENANT_ID") + lr.Attributes().PutStr(azureOperationName, "SecretGet") + lr.Attributes().PutStr(azureOperationVersion, "7.0") + lr.Attributes().PutStr(azureCategory, "AuditEvent") + lr.Attributes().PutStr(azureCorrelationID, guid) + lr.Attributes().PutStr(azureResultType, "Success") + lr.Attributes().PutStr(azureResultSignature, "Signature") + lr.Attributes().PutStr(azureResultDescription, "Description") + lr.Attributes().PutInt(azureDuration, 1234) + lr.Attributes().PutStr(conventions.AttributeNetSockPeerAddr, "127.0.0.1") + lr.Attributes().PutStr(conventions.AttributeCloudRegion, "ukso") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + lr.Attributes().PutEmptyMap(azureIdentity).PutEmptyMap("claim").PutStr("oid", guid) + m := lr.Attributes().PutEmptyMap(azureProperties) + m.PutStr("string", "string") + m.PutDouble("int", 429) + m.PutDouble("float", 3.14) + m.PutBool("bool", false) + + return lr +}() + +var maximumLogRecord2 = func() []plog.LogRecord { + sl := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty() + lr := sl.LogRecords().AppendEmpty() + lr2 := sl.LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2022-11-11T04:48:29.6767145Z") + lr.SetTimestamp(ts) + lr.SetSeverityNumber(plog.SeverityNumberWarn) + lr.SetSeverityText("Warning") + guid := "96317703-2132-4a8d-a5d7-e18d2f486783" + + lr.Attributes().PutStr(azureTenantID, "/TENANT_ID") + lr.Attributes().PutStr(azureOperationName, "SecretSet") + lr.Attributes().PutStr(azureOperationVersion, "7.0") + lr.Attributes().PutStr(azureCategory, "AuditEvent") + lr.Attributes().PutStr(azureCorrelationID, guid) + lr.Attributes().PutStr(azureResultType, "Success") + lr.Attributes().PutStr(azureResultSignature, "Signature") + lr.Attributes().PutStr(azureResultDescription, "Description") + lr.Attributes().PutInt(azureDuration, 4321) + lr.Attributes().PutStr(conventions.AttributeNetSockPeerAddr, "127.0.0.1") + lr.Attributes().PutStr(conventions.AttributeCloudRegion, "ukso") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + lr.Attributes().PutEmptyMap(azureIdentity).PutEmptyMap("claim").PutStr("oid", guid) + m := lr.Attributes().PutEmptyMap(azureProperties) + m.PutStr("string", "string") + m.PutDouble("int", 924) + m.PutDouble("float", 41.3) + m.PutBool("bool", true) + + ts, _ = asTimestamp("2022-11-11T04:48:31.6767145Z") + lr2.SetTimestamp(ts) + lr2.SetSeverityNumber(plog.SeverityNumberWarn) + lr2.SetSeverityText("Warning") + guid = "4ae807da-39d9-4327-b5b4-0ab685a57f9a" + + lr2.Attributes().PutStr(azureTenantID, "/TENANT_ID") + lr2.Attributes().PutStr(azureOperationName, "SecretGet") + lr2.Attributes().PutStr(azureOperationVersion, "7.0") + lr2.Attributes().PutStr(azureCategory, "AuditEvent") + lr2.Attributes().PutStr(azureCorrelationID, guid) + lr2.Attributes().PutStr(azureResultType, "Success") + lr2.Attributes().PutStr(azureResultSignature, "Signature") + lr2.Attributes().PutStr(azureResultDescription, "Description") + lr2.Attributes().PutInt(azureDuration, 321) + lr2.Attributes().PutStr(conventions.AttributeNetSockPeerAddr, "127.0.0.1") + lr2.Attributes().PutStr(conventions.AttributeCloudRegion, "ukso") + lr2.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + lr2.Attributes().PutEmptyMap(azureIdentity).PutEmptyMap("claim").PutStr("oid", guid) + m = lr2.Attributes().PutEmptyMap(azureProperties) + m.PutStr("string", "string") + m.PutDouble("int", 925) + m.PutDouble("float", 41.4) + m.PutBool("bool", false) + + var records []plog.LogRecord + return append(records, lr, lr2) +}() + +var badLevelLogRecord = func() plog.LogRecord { + lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2023-10-26T14:22:43.3416357Z") + lr.SetTimestamp(ts) + lr.SetSeverityNumber(plog.SeverityNumberTrace4) + lr.SetSeverityText("4") + guid := "128bc026-5ead-40c7-8853-ebb32bc077a3" + + lr.Attributes().PutStr(azureOperationName, "Microsoft.ApiManagement/GatewayLogs") + lr.Attributes().PutStr(azureCategory, "GatewayLogs") + lr.Attributes().PutStr(azureCorrelationID, guid) + lr.Attributes().PutStr(azureResultType, "Succeeded") + lr.Attributes().PutInt(azureDuration, 243) + lr.Attributes().PutStr(conventions.AttributeNetSockPeerAddr, "13.14.15.16") + lr.Attributes().PutStr(conventions.AttributeCloudRegion, "West US") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + m := lr.Attributes().PutEmptyMap(azureProperties) + m.PutStr("method", "GET") + m.PutStr("url", "https://api.azure-api.net/sessions") + m.PutDouble("backendResponseCode", 200) + m.PutDouble("responseCode", 200) + m.PutDouble("responseSize", 102945) + m.PutStr("cache", "none") + m.PutDouble("backendTime", 54) + m.PutDouble("requestSize", 632) + m.PutStr("apiId", "demo-api") + m.PutStr("operationId", "GetSessions") + m.PutStr("apimSubscriptionId", "master") + m.PutDouble("clientTime", 190) + m.PutStr("clientProtocol", "HTTP/1.1") + m.PutStr("backendProtocol", "HTTP/1.1") + m.PutStr("apiRevision", "1") + m.PutStr("clientTlsVersion", "1.2") + m.PutStr("backendMethod", "GET") + m.PutStr("backendUrl", "https://api.azurewebsites.net/sessions") + return lr +}() + +var badTimeLogRecord = func() plog.LogRecord { + lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2021-10-14T22:17:11+00:00") + lr.SetTimestamp(ts) + + lr.Attributes().PutStr(azureOperationName, "ApplicationGatewayAccess") + lr.Attributes().PutStr(azureCategory, "ApplicationGatewayAccessLog") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + m := lr.Attributes().PutEmptyMap(azureProperties) + m.PutStr("instanceId", "appgw_2") + m.PutStr("clientIP", "185.42.129.24") + m.PutDouble("clientPort", 45057) + m.PutStr("httpMethod", "GET") + m.PutStr("originalRequestUriWithArgs", "/") + m.PutStr("requestUri", "/") + m.PutStr("requestQuery", "") + m.PutStr("userAgent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36") + m.PutDouble("httpStatus", 200) + m.PutStr("httpVersion", "HTTP/1.1") + m.PutDouble("receivedBytes", 184) + m.PutDouble("sentBytes", 466) + m.PutDouble("clientResponseTime", 0) + m.PutDouble("timeTaken", 0.034) + m.PutStr("WAFEvaluationTime", "0.000") + m.PutStr("WAFMode", "Detection") + m.PutStr("transactionId", "592d1649f75a8d480a3c4dc6a975309d") + m.PutStr("sslEnabled", "on") + m.PutStr("sslCipher", "ECDHE-RSA-AES256-GCM-SHA384") + m.PutStr("sslProtocol", "TLSv1.2") + m.PutStr("sslClientVerify", "NONE") + m.PutStr("sslClientCertificateFingerprint", "") + m.PutStr("sslClientCertificateIssuerName", "") + m.PutStr("serverRouted", "52.239.221.65:443") + m.PutStr("serverStatus", "200") + m.PutStr("serverResponseLatency", "0.028") + m.PutStr("upstreamSourcePort", "21564") + m.PutStr("originalHost", "20.110.30.194") + m.PutStr("host", "20.110.30.194") + return lr +}() + +func TestAsTimestamp(t *testing.T) { + timestamp := "2022-11-11T04:48:27.6767145Z" + nanos, err := asTimestamp(timestamp) + assert.NoError(t, err) + assert.Less(t, pcommon.Timestamp(0), nanos) + + timestamp = "invalid-time" + nanos, err = asTimestamp(timestamp) + assert.Error(t, err) + assert.Equal(t, pcommon.Timestamp(0), nanos) +} + +func TestAsSeverity(t *testing.T) { + tests := map[string]plog.SeverityNumber{ + "Informational": plog.SeverityNumberInfo, + "Warning": plog.SeverityNumberWarn, + "Error": plog.SeverityNumberError, + "Critical": plog.SeverityNumberFatal, + "unknown": plog.SeverityNumberUnspecified, + } + + for input, expected := range tests { + t.Run(input, func(t *testing.T) { + assert.Equal(t, expected, asSeverity(json.Number(input))) + }) + } +} + +func TestSetIf(t *testing.T) { + m := map[string]any{} + + setIf(m, "key", nil) + actual, found := m["key"] + assert.False(t, found) + assert.Nil(t, actual) + + v := "" + setIf(m, "key", &v) + actual, found = m["key"] + assert.False(t, found) + assert.Nil(t, actual) + + v = "ok" + setIf(m, "key", &v) + actual, found = m["key"] + assert.True(t, found) + assert.Equal(t, "ok", actual) +} + +func TestExtractRawAttributes(t *testing.T) { + badDuration := json.Number("invalid") + goodDuration := json.Number("1234") + + tenantID := "tenant.id" + operationVersion := "operation.version" + resultType := "result.type" + resultSignature := "result.signature" + resultDescription := "result.description" + callerIPAddress := "127.0.0.1" + correlationID := "edb70d1a-eec2-4b4c-b2f4-60e3510160ee" + level := json.Number("Informational") + location := "location" + + identity := any("someone") + + properties := any(map[string]any{ + "a": uint64(1), + "b": true, + "c": 1.23, + "d": "ok", + }) + + tests := []struct { + name string + log azureLogRecord + expected map[string]any + }{ + { + name: "minimal", + log: azureLogRecord{ + Time: "", + ResourceID: "resource.id", + OperationName: "operation.name", + Category: "category", + DurationMs: &badDuration, + }, + expected: map[string]any{ + azureOperationName: "operation.name", + azureCategory: "category", + conventions.AttributeCloudProvider: conventions.AttributeCloudProviderAzure, + }, + }, + { + name: "bad-duration", + log: azureLogRecord{ + Time: "", + ResourceID: "resource.id", + OperationName: "operation.name", + Category: "category", + DurationMs: &badDuration, + }, + expected: map[string]any{ + azureOperationName: "operation.name", + azureCategory: "category", + conventions.AttributeCloudProvider: conventions.AttributeCloudProviderAzure, + }, + }, + { + name: "everything", + log: azureLogRecord{ + Time: "", + ResourceID: "resource.id", + TenantID: &tenantID, + OperationName: "operation.name", + OperationVersion: &operationVersion, + Category: "category", + ResultType: &resultType, + ResultSignature: &resultSignature, + ResultDescription: &resultDescription, + DurationMs: &goodDuration, + CallerIPAddress: &callerIPAddress, + CorrelationID: &correlationID, + Identity: &identity, + Level: &level, + Location: &location, + Properties: &properties, + }, + expected: map[string]any{ + azureTenantID: "tenant.id", + azureOperationName: "operation.name", + azureOperationVersion: "operation.version", + azureCategory: "category", + azureCorrelationID: correlationID, + azureResultType: "result.type", + azureResultSignature: "result.signature", + azureResultDescription: "result.description", + azureDuration: int64(1234), + conventions.AttributeNetSockPeerAddr: "127.0.0.1", + azureIdentity: "someone", + conventions.AttributeCloudRegion: "location", + conventions.AttributeCloudProvider: conventions.AttributeCloudProviderAzure, + azureProperties: properties, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, extractRawAttributes(tt.log)) + }) + } + +} + +func TestUnmarshalLogs(t *testing.T) { + expectedMinimum := plog.NewLogs() + resourceLogs := expectedMinimum.ResourceLogs().AppendEmpty() + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr := scopeLogs.LogRecords().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID") + minimumLogRecord.CopyTo(lr) + + expectedMinimum2 := plog.NewLogs() + resourceLogs = expectedMinimum2.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + logRecords := scopeLogs.LogRecords() + lr = logRecords.AppendEmpty() + minimumLogRecord.CopyTo(lr) + lr = logRecords.AppendEmpty() + minimumLogRecord.CopyTo(lr) + + expectedMaximum := plog.NewLogs() + resourceLogs = expectedMaximum.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID-1") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr = scopeLogs.LogRecords().AppendEmpty() + maximumLogRecord1.CopyTo(lr) + + resourceLogs = expectedMaximum.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID-2") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr = scopeLogs.LogRecords().AppendEmpty() + lr2 := scopeLogs.LogRecords().AppendEmpty() + maximumLogRecord2[0].CopyTo(lr) + maximumLogRecord2[1].CopyTo(lr2) + + expectedBadLevel := plog.NewLogs() + resourceLogs = expectedBadLevel.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr = scopeLogs.LogRecords().AppendEmpty() + badLevelLogRecord.CopyTo(lr) + + expectedBadTime := plog.NewLogs() + resourceLogs = expectedBadTime.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr = scopeLogs.LogRecords().AppendEmpty() + badTimeLogRecord.CopyTo(lr) + + tests := []struct { + file string + expected plog.Logs + }{ + { + file: "log-minimum.json", + expected: expectedMinimum, + }, + { + file: "log-minimum-2.json", + expected: expectedMinimum2, + }, + { + file: "log-maximum.json", + expected: expectedMaximum, + }, + { + file: "log-bad-level.json", + expected: expectedBadLevel, + }, + { + file: "log-bad-time.json", + expected: expectedBadTime, + }, + } + + sut := &ResourceLogsUnmarshaler{ + Version: testBuildInfo.Version, + Logger: zap.NewNop(), + } + for _, tt := range tests { + t.Run(tt.file, func(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", tt.file)) + assert.NoError(t, err) + assert.NotNil(t, data) + + logs, err := sut.UnmarshalLogs(data) + assert.NoError(t, err) + + assert.NoError(t, plogtest.CompareLogs(tt.expected, logs)) + }) + } +} + +func loadJsonLogsAndApplySemanticConventions(filename string) (plog.Logs, error) { + l := plog.NewLogs() + + sut := &ResourceLogsUnmarshaler{ + Version: testBuildInfo.Version, + Logger: zap.NewNop(), + } + + data, err := os.ReadFile(filepath.Join("testdata", filename)) + if err != nil { + return l, err + } + + logs, err := sut.UnmarshalLogs(data) + + if err != nil { + return l, err + } + + return logs, nil +} + +func TestAzureCdnAccessLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-azurecdnaccesslog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, int64(1234), record["http.request.size"]) + assert.Equal(t, int64(12345), record["http.response.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "tls", record["tls.protocol.name"]) + assert.Equal(t, "1.3", record["tls.protocol.version"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "NoError", record["error.type"]) +} + +func TestFrontDoorAccessLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdooraccesslog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, int64(1234), record["http.request.size"]) + assert.Equal(t, int64(12345), record["http.response.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "23.23.23.23", record["network.peer.address"]) + assert.Equal(t, float64(0.23), record["http.server.request.duration"]) + assert.Equal(t, "https", record["network.protocol.name"]) + assert.Equal(t, "tls", record["tls.protocol.name"]) + assert.Equal(t, "1.3", record["tls.protocol.version"]) + assert.Equal(t, "TLS_AES_256_GCM_SHA384", record["tls.cipher"]) + assert.Equal(t, "secp384r1", record["tls.curve"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "REFERER", record["http.request.header.referer"]) + assert.Equal(t, "NoError", record["error.type"]) +} + +func TestFrontDoorHealthProbeLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdoorhealthprobelog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "https://probe.net/health", record["url.full"]) + assert.Equal(t, "42.42.42.42", record["server.address"]) + assert.Equal(t, 0.042, record["http.request.duration"]) + assert.Equal(t, 0.00023, record["dns.lookup.duration"]) +} + +func TestFrontDoorWAFLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdoorwaflog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, "test.net", record["server.address"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "23.23.23.23", record["network.peer.address"]) +} + +func TestAppServiceAppLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceapplogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "EXCEPTION_CLASS", record["exception.type"]) + assert.Equal(t, "HOST", record["host.id"]) + assert.Equal(t, "METHOD", record["code.function"]) + assert.Equal(t, "FILEPATH", record["code.filepath"]) + assert.Equal(t, "STACKTRACE", record["exception.stacktrace"]) +} + +func TestAppServiceConsoleLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceconsolelogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "HOST", record["host.id"]) +} + +func TestAppServiceAuditLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceauditlogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "USER_ID", record["enduser.id"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "kudu", record["network.protocol.name"]) +} + +func TestAppServiceHTTPLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appservicehttplogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "test.com", record["url.domain"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, int64(80), record["server.port"]) + assert.Equal(t, "/api/test/", record["url.path"]) + assert.Equal(t, "foo=42", record["url.query"]) + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, 0.42, record["http.server.request.duration"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, int64(4242), record["http.request.body.size"]) + assert.Equal(t, int64(42), record["http.response.body.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "REFERER", record["http.request.header.referer"]) + assert.Equal(t, "COMPUTER_NAME", record["host.name"]) + assert.Equal(t, "http", record["network.protocol.name"]) + assert.Equal(t, "1.1", record["network.protocol.version"]) +} + +func TestAppServicePlatformLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceplatformlogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "CONTAINER_NAME", record["container.name"]) +} + +func TestAppServiceIPSecAuditLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceipsecauditlogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "HOST", record["url.domain"]) + assert.Equal(t, "FDID", record["http.request.header.x-azure-fdid"]) + assert.Equal(t, "HEALTH_PROBE", record["http.request.header.x-fd-healthprobe"]) + assert.Equal(t, "FORWARDED_FOR", record["http.request.header.x-forwarded-for"]) + assert.Equal(t, "FORWARDED_HOST", record["http.request.header.x-forwarded-host"]) +} diff --git a/pkg/translator/azure_logs/testdata/log-appserviceapplogs.json b/pkg/translator/azure_logs/testdata/log-appserviceapplogs.json new file mode 100644 index 000000000000..202a755a2286 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-appserviceapplogs.json @@ -0,0 +1,18 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "AppServiceAppLogs", + "operationName": "AppLog", + "properties": { + "ContainerId": "CONTAINER_ID", + "ExceptionClass": "EXCEPTION_CLASS", + "Host": "HOST", + "Method": "METHOD", + "Source": "FILEPATH", + "Stacktrace": "STACKTRACE" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-appserviceauditlogs.json b/pkg/translator/azure_logs/testdata/log-appserviceauditlogs.json new file mode 100644 index 000000000000..ba64020d5954 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-appserviceauditlogs.json @@ -0,0 +1,16 @@ +{ + "records": [ + { + "time": "2024-04-24T12:01:20.8427400Z", + "ResourceId": "/SUBSCRIPTIONS/DA2DD5CC-E7BC-4DB6-94D9-0AFB3BD30577/RESOURCEGROUPS/FRETBADGER/PROVIDERS/MICROSOFT.WEB/SITES/FBEHTESTAPP", + "Category": "AppServiceAuditLogs", + "OperationName": "Authorization", + "Properties": { + "User": "USER_ID", + "UserDisplayName": "$fbehtestapp", + "UserAddress": "42.42.42.42", + "Protocol": "Kudu" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-appserviceconsolelogs.json b/pkg/translator/azure_logs/testdata/log-appserviceconsolelogs.json new file mode 100644 index 000000000000..8158b619f8a8 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-appserviceconsolelogs.json @@ -0,0 +1,14 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "AppServiceConsoleLogs", + "operationName": "ConsoleLog", + "properties": { + "ContainerId": "CONTAINER_ID", + "Host": "HOST" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-appservicehttplogs.json b/pkg/translator/azure_logs/testdata/log-appservicehttplogs.json new file mode 100644 index 000000000000..6565dc976b3b --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-appservicehttplogs.json @@ -0,0 +1,34 @@ +{ + "records": [ + { + "time": "2024-04-24T11:59:40.9893370Z", + "EventTime": "2024-04-24T11:59:40.9893370Z", + "resourceId": "/SUBSCRIPTIONS/DA2DD5CC-E7BC-4DB6-94D9-0AFB3BD30577/RESOURCEGROUPS/FRETBADGER/PROVIDERS/MICROSOFT.WEB/SITES/FBEHTESTAPP", + "properties": { + "CsHost": "test.com", + "CIp": "42.42.42.42", + "SPort": "80", + "CsUriStem": "/api/test/", + "CsUriQuery": "foo=42", + "CsMethod": "GET", + "TimeTaken": 420, + "ScStatus": "200", + "Result": "Success", + "CsBytes": "4242", + "ScBytes": "42", + "UserAgent": "Mozilla/5.0", + "Cookie": "", + "CsUsername": "user@test.com", + "Referer": "REFERER", + "ComputerName": "COMPUTER_NAME", + "Protocol": "HTTP/1.1" + }, + "category": "AppServiceHTTPLogs", + "EventStampType": "Stamp", + "EventPrimaryStampName": "waws-prod-blu-479", + "EventStampName": "waws-prod-blu-479", + "Host": "lw0sdlwk0005XR", + "EventIpAddress": "10.50.0.34" + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-appserviceipsecauditlogs.json b/pkg/translator/azure_logs/testdata/log-appserviceipsecauditlogs.json new file mode 100644 index 000000000000..207d81ff629d --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-appserviceipsecauditlogs.json @@ -0,0 +1,18 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "AppServiceIPSecAuditLogs", + "operationName": "IPSecAuditLog", + "properties": { + "CIp": "42.42.42.42", + "CsHost": "HOST", + "XAzureFDID": "FDID", + "XFDHealthProbe": "HEALTH_PROBE", + "XForwardedFor": "FORWARDED_FOR", + "XForwardedHost": "FORWARDED_HOST" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-appserviceplatformlogs.json b/pkg/translator/azure_logs/testdata/log-appserviceplatformlogs.json new file mode 100644 index 000000000000..903e5dcfcd25 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-appserviceplatformlogs.json @@ -0,0 +1,21 @@ +{ + "records": [ + { + "resourceId": "/SUBSCRIPTIONS/DA2DD5CC-E7BC-4DB6-94D9-0AFB3BD30577/RESOURCEGROUPS/FRETBADGER/PROVIDERS/MICROSOFT.WEB/SITES/FBEHTESTAPP", + "category": "AppServicePlatformLogs", + "time": "2024-04-24T12:03:55.630Z", + "level": "Informational", + "operationName": "ContainerLogs", + "properties": { + "message": "Initiating warmup request to container fbehtestapp_1_b8a27b37 for site fbehtestapp", + "containerId": "CONTAINER_ID", + "containerName": "CONTAINER_NAME" + }, + "EventStampType": "Stamp", + "EventPrimaryStampName": "waws-prod-blu-479", + "EventStampName": "waws-prod-blu-479", + "Host": "lw0sdlwk0005XR", + "EventIpAddress": "10.50.0.34" + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-azurecdnaccesslog.json b/pkg/translator/azure_logs/testdata/log-azurecdnaccesslog.json new file mode 100644 index 000000000000..2a905fb85e16 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-azurecdnaccesslog.json @@ -0,0 +1,34 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "AzureCdnAccessLog", + "operationName": "Microsoft.AzureCdn/Profiles/AccessLog", + "properties": { + "BackendHostName": "backendhost.net", + "ClientIp": "42.42.42.42", + "ClientPort": "0", + "HttpMethod": "GET", + "HttpVersion": "1.1.0.0", + "HttpStatusCode": "200", + "HttpStatusDetails": "200", + "POP": "LON", + "RequestBytes": "1234", + "RequestUri": "https://test.net/", + "ResponseBytes": "12345", + "RoutingRuleName": "default-route", + "RulesEngineMatchNames": [], + "SecurityProtocol": "TLS 1.3", + "isReceivedFromClient": false, + "TimeTaken": "0.230", + "TrackingReference": "TRACKING_REFERENCE", + "UserAgent": "Mozilla/5.0", + "ErrorInfo": "NoError", + "TimeToFirstByte": "0.420", + "Result": "N/A", + "SNI": "originshield|parentcache|https|tier2" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-bad-level.json b/pkg/translator/azure_logs/testdata/log-bad-level.json new file mode 100644 index 000000000000..662d34821f28 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-bad-level.json @@ -0,0 +1,39 @@ +{ + "records": [ + { + "DeploymentVersion": "0.40.16708.0", + "Level": 4, + "isRequestSuccess": true, + "time": "2023-10-26T14:22:43.3416357Z", + "operationName": "Microsoft.ApiManagement/GatewayLogs", + "category": "GatewayLogs", + "durationMs": 243, + "callerIpAddress": "13.14.15.16", + "correlationId": "128bc026-5ead-40c7-8853-ebb32bc077a3", + "location": "West US", + "properties": { + "method": "GET", + "url": "https://api.azure-api.net/sessions", + "backendResponseCode": 200, + "responseCode": 200, + "responseSize": 102945, + "cache": "none", + "backendTime": 54, + "requestSize": 632, + "apiId": "demo-api", + "operationId": "GetSessions", + "apimSubscriptionId": "master", + "clientTime": 190, + "clientProtocol": "HTTP/1.1", + "backendProtocol": "HTTP/1.1", + "apiRevision": "1", + "clientTlsVersion": "1.2", + "backendMethod": "GET", + "backendUrl": "https://api.azurewebsites.net/sessions" + }, + "resourceId": "/RESOURCE_ID", + "resultType": "Succeeded", + "truncated": 0 + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-bad-time.json b/pkg/translator/azure_logs/testdata/log-bad-time.json new file mode 100644 index 000000000000..614d170378ec --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-bad-time.json @@ -0,0 +1,45 @@ +{ + "records": [ + { + "timeStamp": "2021-10-14T22:17:11+00:00", + "resourceId": "/RESOURCE_ID", + "listenerName": "HTTP-Listener", + "ruleName": "Storage-Static-Rule", + "backendPoolName": "StaticStorageAccount", + "backendSettingName": "StorageStatic-HTTPS-Setting", + "operationName": "ApplicationGatewayAccess", + "category": "ApplicationGatewayAccessLog", + "properties": { + "instanceId": "appgw_2", + "clientIP": "185.42.129.24", + "clientPort": 45057, + "httpMethod": "GET", + "originalRequestUriWithArgs": "\/", + "requestUri": "\/", + "requestQuery": "", + "userAgent": "Mozilla\/5.0 (Windows NT 6.1; WOW64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/52.0.2743.116 Safari\/537.36", + "httpStatus": 200, + "httpVersion": "HTTP\/1.1", + "receivedBytes": 184, + "sentBytes": 466, + "clientResponseTime": 0, + "timeTaken": 0.034, + "WAFEvaluationTime": "0.000", + "WAFMode": "Detection", + "transactionId": "592d1649f75a8d480a3c4dc6a975309d", + "sslEnabled": "on", + "sslCipher": "ECDHE-RSA-AES256-GCM-SHA384", + "sslProtocol": "TLSv1.2", + "sslClientVerify": "NONE", + "sslClientCertificateFingerprint": "", + "sslClientCertificateIssuerName": "", + "serverRouted": "52.239.221.65:443", + "serverStatus": "200", + "serverResponseLatency": "0.028", + "upstreamSourcePort": "21564", + "originalHost": "20.110.30.194", + "host": "20.110.30.194" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-frontdooraccesslog.json b/pkg/translator/azure_logs/testdata/log-frontdooraccesslog.json new file mode 100644 index 000000000000..0a775bdb9572 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-frontdooraccesslog.json @@ -0,0 +1,46 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "FrontDoorAccessLog", + "operationName": "Microsoft.Cdn/Profiles/AccessLog/Write", + "properties": { + "trackingReference": "TRACKING_REFERENCE", + "httpMethod": "GET", + "httpVersion": "1.1.0.0", + "requestUri": "https://test.net/", + "sni": "originshield|parentcache|https|tier2", + "requestBytes": "1234", + "responseBytes": "12345", + "userAgent": "Mozilla/5.0", + "clientIp": "42.42.42.42", + "clientPort": "0", + "socketIp": "23.23.23.23", + "timeToFirstByte": "0.420", + "timeTaken": "0.230", + "requestProtocol": "HTTPS", + "securityProtocol": "TLS 1.3", + "rulesEngineMatchNames": [], + "httpStatusCode": "200", + "httpStatusDetails": "200", + "pop": "LON", + "cacheStatus": "MISS", + "errorInfo": "NoError", + "ErrorInfo": "NoError", + "result": "N/A", + "endpoint": "dummyapp-eebde0bwehfthfbb.z01.azurefd.net", + "routingRuleName": "default-route", + "hostName": "dummyapp-eebde0bwehfthfbb.z01.azurefd.net", + "originUrl": "https://dummyapp.icysea-e3cd8b77.eastus.azurecontainerapps.io:443/", + "originIp": "4.156.233.180:443", + "originName": "dummyapp.icysea-e3cd8b77.eastus.azurecontainerapps.io:443", + "referer": "REFERER", + "clientCountry": "United Kingdom", + "domain": "dummyapp-eebde0bwehfthfbb.z01.azurefd.net:443", + "securityCipher": "TLS_AES_256_GCM_SHA384", + "securityCurves": "secp384r1" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-frontdoorhealthprobelog.json b/pkg/translator/azure_logs/testdata/log-frontdoorhealthprobelog.json new file mode 100644 index 000000000000..f0cf4bb0560e --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-frontdoorhealthprobelog.json @@ -0,0 +1,22 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "FrontDoorHealthProbeLog", + "operationName": "WAF/FirewallLog", + "properties": { + "healthProbeId": "AAAA", + "POP": "", + "httpVerb": "GET", + "result": "", + "httpStatusCode": "200", + "probeURL": "https://probe.net/health", + "originName": "https://probe.net/", + "originIP": "42.42.42.42", + "totalLatencyMilliseconds": "42", + "DNSLatencyMicroseconds": "230" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-frontdoorwaflog.json b/pkg/translator/azure_logs/testdata/log-frontdoorwaflog.json new file mode 100644 index 000000000000..6a894fe955a7 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-frontdoorwaflog.json @@ -0,0 +1,18 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "FrontdoorWebApplicationFirewallLog", + "operationName": "WAF/FirewallLog", + "properties": { + "trackingReference": "TRACKING_REFERENCE", + "clientIP": "42.42.42.42", + "clientPort": "0", + "socketIP": "23.23.23.23", + "requestUri": "https://test.net/", + "host": "test.net" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-maximum.json b/pkg/translator/azure_logs/testdata/log-maximum.json new file mode 100644 index 000000000000..b105c6b168bc --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-maximum.json @@ -0,0 +1,85 @@ +{ + "records": [ + { + "time": "2022-11-11T04:48:27.6767145Z", + "resourceId": "/RESOURCE_ID-1", + "tenantId": "/TENANT_ID", + "operationName": "SecretGet", + "operationVersion": "7.0", + "category": "AuditEvent", + "resultType": "Success", + "resultSignature": "Signature", + "resultDescription": "Description", + "durationMs": "1234", + "callerIpAddress": "127.0.0.1", + "correlationId": "607964b6-41a5-4e24-a5db-db7aab3b9b34", + "Level": "Warning", + "location": "ukso", + "identity": { + "claim": { + "oid": "607964b6-41a5-4e24-a5db-db7aab3b9b34" + } + }, + "properties": { + "string": "string", + "int": 429, + "float": 3.14, + "bool": false + } + }, + { + "time": "2022-11-11T04:48:29.6767145Z", + "resourceId": "/RESOURCE_ID-2", + "tenantId": "/TENANT_ID", + "operationName": "SecretSet", + "operationVersion": "7.0", + "category": "AuditEvent", + "resultType": "Success", + "resultSignature": "Signature", + "resultDescription": "Description", + "durationMs": "4321", + "callerIpAddress": "127.0.0.1", + "correlationId": "96317703-2132-4a8d-a5d7-e18d2f486783", + "Level": "Warning", + "location": "ukso", + "identity": { + "claim": { + "oid": "96317703-2132-4a8d-a5d7-e18d2f486783" + } + }, + "properties": { + "string": "string", + "int": 924, + "float": 41.3, + "bool": true + } + }, + { + "time": "2022-11-11T04:48:31.6767145Z", + "resourceId": "/RESOURCE_ID-2", + "tenantId": "/TENANT_ID", + "operationName": "SecretGet", + "operationVersion": "7.0", + "category": "AuditEvent", + "resultType": "Success", + "resultSignature": "Signature", + "resultDescription": "Description", + "durationMs": "321", + "callerIpAddress": "127.0.0.1", + "correlationId": "4ae807da-39d9-4327-b5b4-0ab685a57f9a", + "Level": "Warning", + "location": "ukso", + "identity": { + "claim": { + "oid": "4ae807da-39d9-4327-b5b4-0ab685a57f9a" + } + }, + "properties": { + "string": "string", + "int": 925, + "float": 41.4, + "bool": false + } + } + ] +} diff --git a/pkg/translator/azure_logs/testdata/log-minimum-2.json b/pkg/translator/azure_logs/testdata/log-minimum-2.json new file mode 100644 index 000000000000..6eac63fa0389 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-minimum-2.json @@ -0,0 +1,16 @@ +{ + "records": [ + { + "time": "2022-11-11T04:48:27.6767145Z", + "resourceId": "/RESOURCE_ID", + "operationName": "SecretGet", + "category": "AuditEvent" + }, + { + "time": "2022-11-11T04:48:27.6767145Z", + "resourceId": "/RESOURCE_ID", + "operationName": "SecretGet", + "category": "AuditEvent" + } + ] +} diff --git a/pkg/translator/azure_logs/testdata/log-minimum.json b/pkg/translator/azure_logs/testdata/log-minimum.json new file mode 100644 index 000000000000..16d4f2e71177 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-minimum.json @@ -0,0 +1,10 @@ +{ + "records": [ + { + "time": "2022-11-11T04:48:27.6767145Z", + "resourceId": "/RESOURCE_ID", + "operationName": "SecretGet", + "category": "AuditEvent" + } + ] +} diff --git a/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go b/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go index 1665c9f2d833..e6971f1dbc8d 100644 --- a/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go +++ b/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go @@ -10,19 +10,33 @@ import ( "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs" ) -type AzureResourceLogsEventUnmarshaler struct { - unmarshaler *azure.ResourceLogsUnmarshaler +type logsUnmarshaler interface { + UnmarshalLogs([]byte) (plog.Logs, error) } -func newAzureResourceLogsUnmarshaler(buildInfo component.BuildInfo, logger *zap.Logger) eventLogsUnmarshaler { +type AzureResourceLogsEventUnmarshaler struct { + unmarshaler logsUnmarshaler +} - return AzureResourceLogsEventUnmarshaler{ - unmarshaler: &azure.ResourceLogsUnmarshaler{ - Version: buildInfo.Version, - Logger: logger, - }, +func newAzureResourceLogsUnmarshaler(buildInfo component.BuildInfo, logger *zap.Logger, applySemanticConventions bool) eventLogsUnmarshaler { + + if applySemanticConventions { + return AzureResourceLogsEventUnmarshaler{ + unmarshaler: &azure_logs.ResourceLogsUnmarshaler{ + Version: buildInfo.Version, + Logger: logger, + }, + } + } else { + return AzureResourceLogsEventUnmarshaler{ + unmarshaler: &azure.ResourceLogsUnmarshaler{ + Version: buildInfo.Version, + Logger: logger, + }, + } } } diff --git a/receiver/azureeventhubreceiver/config.go b/receiver/azureeventhubreceiver/config.go index 9ccb99205fc8..e5e01c98cf87 100644 --- a/receiver/azureeventhubreceiver/config.go +++ b/receiver/azureeventhubreceiver/config.go @@ -25,12 +25,13 @@ var ( ) type Config struct { - Connection string `mapstructure:"connection"` - Partition string `mapstructure:"partition"` - Offset string `mapstructure:"offset"` - StorageID *component.ID `mapstructure:"storage"` - Format string `mapstructure:"format"` - ConsumerGroup string `mapstructure:"group"` + Connection string `mapstructure:"connection"` + Partition string `mapstructure:"partition"` + Offset string `mapstructure:"offset"` + StorageID *component.ID `mapstructure:"storage"` + Format string `mapstructure:"format"` + ConsumerGroup string `mapstructure:"group"` + ApplySemanticConventions bool `mapstructure:"apply_semantic_conventions"` } func isValidFormat(format string) bool { diff --git a/receiver/azureeventhubreceiver/config_test.go b/receiver/azureeventhubreceiver/config_test.go index 7d4dcf4c546f..36497f0d8e13 100644 --- a/receiver/azureeventhubreceiver/config_test.go +++ b/receiver/azureeventhubreceiver/config_test.go @@ -33,12 +33,15 @@ func TestLoadConfig(t *testing.T) { assert.Equal(t, "", r0.(*Config).Offset) assert.Equal(t, "", r0.(*Config).Partition) assert.Equal(t, defaultLogFormat, logFormat(r0.(*Config).Format)) + assert.False(t, r0.(*Config).ApplySemanticConventions) r1 := cfg.Receivers[component.NewIDWithName(metadata.Type, "all")] assert.Equal(t, "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=superSecret1234=;EntityPath=hubName", r1.(*Config).Connection) assert.Equal(t, "1234-5566", r1.(*Config).Offset) assert.Equal(t, "foo", r1.(*Config).Partition) assert.Equal(t, rawLogFormat, logFormat(r1.(*Config).Format)) + assert.True(t, r1.(*Config).ApplySemanticConventions) + } func TestMissingConnection(t *testing.T) { diff --git a/receiver/azureeventhubreceiver/factory.go b/receiver/azureeventhubreceiver/factory.go index 0e95b8f0e840..83d363299386 100644 --- a/receiver/azureeventhubreceiver/factory.go +++ b/receiver/azureeventhubreceiver/factory.go @@ -100,7 +100,7 @@ func (f *eventhubReceiverFactory) getReceiver( if logFormat(receiverConfig.Format) == rawLogFormat { logsUnmarshaler = newRawLogsUnmarshaler(settings.Logger) } else { - logsUnmarshaler = newAzureResourceLogsUnmarshaler(settings.BuildInfo, settings.Logger) + logsUnmarshaler = newAzureResourceLogsUnmarshaler(settings.BuildInfo, settings.Logger, receiverConfig.ApplySemanticConventions) } case component.DataTypeMetrics: if logFormat(receiverConfig.Format) == rawLogFormat { diff --git a/receiver/azureeventhubreceiver/go.mod b/receiver/azureeventhubreceiver/go.mod index 8333a20674a1..0cba394a1336 100644 --- a/receiver/azureeventhubreceiver/go.mod +++ b/receiver/azureeventhubreceiver/go.mod @@ -9,6 +9,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.99.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.99.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.99.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs v0.99.0 github.com/relvacode/iso8601 v1.4.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.99.1-0.20240503221155-67d37183e6ac @@ -112,7 +113,7 @@ require ( go.opentelemetry.io/proto/otlp v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.22.0 // indirect - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -144,6 +145,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/share replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure => ../../pkg/translator/azure +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs => ../../pkg/translator/azure_logs + replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../pkg/golden replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/common => ../../internal/common diff --git a/receiver/azureeventhubreceiver/go.sum b/receiver/azureeventhubreceiver/go.sum index 4c016e8f9b17..dbaf6601bea2 100644 --- a/receiver/azureeventhubreceiver/go.sum +++ b/receiver/azureeventhubreceiver/go.sum @@ -299,8 +299,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= diff --git a/receiver/azureeventhubreceiver/testdata/config.yaml b/receiver/azureeventhubreceiver/testdata/config.yaml index 4afddc4c7175..091b2cbe0a48 100644 --- a/receiver/azureeventhubreceiver/testdata/config.yaml +++ b/receiver/azureeventhubreceiver/testdata/config.yaml @@ -7,6 +7,7 @@ receivers: partition: foo offset: "1234-5566" format: "raw" + apply_semantic_conventions: true processors: nop: diff --git a/versions.yaml b/versions.yaml index 553569f6852b..1ffb85162f93 100644 --- a/versions.yaml +++ b/versions.yaml @@ -142,6 +142,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure + - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/loki - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus