From 8e68881164baa84890d4508598a1b14f2e1afc07 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Wed, 16 Oct 2024 22:30:07 -0700 Subject: [PATCH 01/21] [receiver/systemd] New receiver for systemd (#35822) #### Description Add a new receiver to handle systemd service state reporting as metrics. #### Link to tracking issue https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33532 --- .chloggen/initial_systemdreceiver.yaml | 27 ++++ .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/bug_report.yaml | 1 + .github/ISSUE_TEMPLATE/feature_request.yaml | 1 + .github/ISSUE_TEMPLATE/other.yaml | 1 + .github/ISSUE_TEMPLATE/unmaintained.yaml | 1 + cmd/githubgen/allowlist.txt | 3 +- receiver/systemdreceiver/Makefile | 1 + receiver/systemdreceiver/README.md | 13 ++ receiver/systemdreceiver/config.go | 6 + receiver/systemdreceiver/doc.go | 6 + receiver/systemdreceiver/factory.go | 35 +++++ .../generated_component_test.go | 69 +++++++++ .../systemdreceiver/generated_package_test.go | 13 ++ receiver/systemdreceiver/go.mod | 58 ++++++++ receiver/systemdreceiver/go.sum | 134 ++++++++++++++++++ .../internal/metadata/generated_status.go | 16 +++ receiver/systemdreceiver/metadata.yaml | 11 ++ receiver/systemdreceiver/receiver.go | 20 +++ versions.yaml | 1 + 20 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 .chloggen/initial_systemdreceiver.yaml create mode 100644 receiver/systemdreceiver/Makefile create mode 100644 receiver/systemdreceiver/README.md create mode 100644 receiver/systemdreceiver/config.go create mode 100644 receiver/systemdreceiver/doc.go create mode 100644 receiver/systemdreceiver/factory.go create mode 100644 receiver/systemdreceiver/generated_component_test.go create mode 100644 receiver/systemdreceiver/generated_package_test.go create mode 100644 receiver/systemdreceiver/go.mod create mode 100644 receiver/systemdreceiver/go.sum create mode 100644 receiver/systemdreceiver/internal/metadata/generated_status.go create mode 100644 receiver/systemdreceiver/metadata.yaml create mode 100644 receiver/systemdreceiver/receiver.go diff --git a/.chloggen/initial_systemdreceiver.yaml b/.chloggen/initial_systemdreceiver.yaml new file mode 100644 index 000000000000..3dd9232d6795 --- /dev/null +++ b/.chloggen/initial_systemdreceiver.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: new_component + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: systemdreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Introduce the scaffolding of a new component, systemdreceiver + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33532] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1826422483ae..37c7aec51903 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -295,6 +295,7 @@ receiver/sqlserverreceiver/ @open-teleme receiver/sshcheckreceiver/ @open-telemetry/collector-contrib-approvers @nslaughter @codeboten receiver/statsdreceiver/ @open-telemetry/collector-contrib-approvers @jmacd @dmitryax receiver/syslogreceiver/ @open-telemetry/collector-contrib-approvers @djaglowski @andrzej-stencel +receiver/systemdreceiver/ @open-telemetry/collector-contrib-approvers @Hemansh31 @atoulme receiver/tcplogreceiver/ @open-telemetry/collector-contrib-approvers @djaglowski receiver/tlscheckreceiver/ @open-telemetry/collector-contrib-approvers @atoulme @michael-burt receiver/udplogreceiver/ @open-telemetry/collector-contrib-approvers @djaglowski diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 359cb4a6b06a..9d736dadb9f8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -290,6 +290,7 @@ body: - receiver/sshcheck - receiver/statsd - receiver/syslog + - receiver/systemd - receiver/tcplog - receiver/tlscheck - receiver/udplog diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 5203d2e7659b..3fd840e735d9 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -284,6 +284,7 @@ body: - receiver/sshcheck - receiver/statsd - receiver/syslog + - receiver/systemd - receiver/tcplog - receiver/tlscheck - receiver/udplog diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml index 11380d858eaa..e21e52dbe341 100644 --- a/.github/ISSUE_TEMPLATE/other.yaml +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -284,6 +284,7 @@ body: - receiver/sshcheck - receiver/statsd - receiver/syslog + - receiver/systemd - receiver/tcplog - receiver/tlscheck - receiver/udplog diff --git a/.github/ISSUE_TEMPLATE/unmaintained.yaml b/.github/ISSUE_TEMPLATE/unmaintained.yaml index ef4baccf3a5c..cf6c3dd7dd46 100644 --- a/.github/ISSUE_TEMPLATE/unmaintained.yaml +++ b/.github/ISSUE_TEMPLATE/unmaintained.yaml @@ -289,6 +289,7 @@ body: - receiver/sshcheck - receiver/statsd - receiver/syslog + - receiver/systemd - receiver/tcplog - receiver/tlscheck - receiver/udplog diff --git a/cmd/githubgen/allowlist.txt b/cmd/githubgen/allowlist.txt index 6ba509caff99..27764783fa17 100644 --- a/cmd/githubgen/allowlist.txt +++ b/cmd/githubgen/allowlist.txt @@ -23,4 +23,5 @@ m1rp jriguera abhishek-at-cloudwerx joker-star-l -michael-burt \ No newline at end of file +michael-burt +Hemansh31 \ No newline at end of file diff --git a/receiver/systemdreceiver/Makefile b/receiver/systemdreceiver/Makefile new file mode 100644 index 000000000000..ded7a36092dc --- /dev/null +++ b/receiver/systemdreceiver/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/receiver/systemdreceiver/README.md b/receiver/systemdreceiver/README.md new file mode 100644 index 000000000000..77125ff8500d --- /dev/null +++ b/receiver/systemdreceiver/README.md @@ -0,0 +1,13 @@ +# Systemd Receiver + + +| Status | | +| ------------- |-----------| +| Stability | [development]: metrics | +| Distributions | [] | +| Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fsystemd%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fsystemd) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fsystemd%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fsystemd) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@Hemansh31](https://www.github.com/Hemansh31), [@atoulme](https://www.github.com/atoulme) | + +[development]: https://github.com/open-telemetry/opentelemetry-collector#development + + diff --git a/receiver/systemdreceiver/config.go b/receiver/systemdreceiver/config.go new file mode 100644 index 000000000000..999c1fbc54f6 --- /dev/null +++ b/receiver/systemdreceiver/config.go @@ -0,0 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package systemdreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/systemdreceiver" + +type Config struct{} diff --git a/receiver/systemdreceiver/doc.go b/receiver/systemdreceiver/doc.go new file mode 100644 index 000000000000..b4d6bd5454d3 --- /dev/null +++ b/receiver/systemdreceiver/doc.go @@ -0,0 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mdatagen metadata.yaml + +package systemdreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/systemdreceiver" diff --git a/receiver/systemdreceiver/factory.go b/receiver/systemdreceiver/factory.go new file mode 100644 index 000000000000..0891b5652978 --- /dev/null +++ b/receiver/systemdreceiver/factory.go @@ -0,0 +1,35 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package systemdreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/systemdreceiver" + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/systemdreceiver/internal/metadata" +) + +// NewFactory creates a factory for systemd receiver. +func NewFactory() receiver.Factory { + return receiver.NewFactory( + metadata.Type, + createDefaultConfig, + receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability)) +} + +func createDefaultConfig() component.Config { + return &Config{} +} + +func createMetricsReceiver( + _ context.Context, + _ receiver.Settings, + _ component.Config, + _ consumer.Metrics, +) (receiver.Metrics, error) { + return systemdReceiver{}, nil +} diff --git a/receiver/systemdreceiver/generated_component_test.go b/receiver/systemdreceiver/generated_component_test.go new file mode 100644 index 000000000000..c8f47c3f98e1 --- /dev/null +++ b/receiver/systemdreceiver/generated_component_test.go @@ -0,0 +1,69 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package systemdreceiver + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, "systemd", NewFactory().Type().String()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + tests := []struct { + name string + createFn func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) + }{ + + { + name: "metrics", + createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateMetrics(ctx, set, cfg, consumertest.NewNop()) + }, + }, + } + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + + for _, tt := range tests { + t.Run(tt.name+"-shutdown", func(t *testing.T) { + c, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run(tt.name+"-lifecycle", func(t *testing.T) { + firstRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + host := componenttest.NewNopHost() + require.NoError(t, err) + require.NoError(t, firstRcvr.Start(context.Background(), host)) + require.NoError(t, firstRcvr.Shutdown(context.Background())) + secondRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + require.NoError(t, secondRcvr.Start(context.Background(), host)) + require.NoError(t, secondRcvr.Shutdown(context.Background())) + }) + } +} diff --git a/receiver/systemdreceiver/generated_package_test.go b/receiver/systemdreceiver/generated_package_test.go new file mode 100644 index 000000000000..ceddbf1a2b34 --- /dev/null +++ b/receiver/systemdreceiver/generated_package_test.go @@ -0,0 +1,13 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package systemdreceiver + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/receiver/systemdreceiver/go.mod b/receiver/systemdreceiver/go.mod new file mode 100644 index 000000000000..6f2283538bf7 --- /dev/null +++ b/receiver/systemdreceiver/go.mod @@ -0,0 +1,58 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/systemdreceiver + +go 1.22.0 + +require ( + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/confmap v1.17.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/consumer v0.111.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/receiver v0.111.1-0.20241008154146-ea48c09c31ae + go.uber.org/goleak v1.3.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/consumer/consumerprofiles v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/internal/globalsignal v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/pipeline v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/receiver/receiverprofiles v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +retract ( + v0.76.2 + v0.76.1 + v0.65.0 +) diff --git a/receiver/systemdreceiver/go.sum b/receiver/systemdreceiver/go.sum new file mode 100644 index 000000000000..1dea2d23ea02 --- /dev/null +++ b/receiver/systemdreceiver/go.sum @@ -0,0 +1,134 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= +github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae h1:dXAMqXGJp1vWG7qwS/2sjIyJgmyOSfEOm6Gcmkzp1cQ= +go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:iWUfPxpVwZhkI4v3/Gh5wt4iKyJn4lriPFAug8iLXno= +go.opentelemetry.io/collector/config/configtelemetry v0.111.1-0.20241008154146-ea48c09c31ae h1:NmNYRBSP+IUK9CsU1Q/1eS/tXmYTPMYxmGQsxEprq/s= +go.opentelemetry.io/collector/config/configtelemetry v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:R0MBUxjSMVMIhljuDHWIygzzJWQyZHXXWIgQNxcFwhc= +go.opentelemetry.io/collector/confmap v1.17.1-0.20241008154146-ea48c09c31ae h1:mYgomPNE0dq1SU1OVnMb/Z8Xbj89jBXnjJexz9M71t0= +go.opentelemetry.io/collector/confmap v1.17.1-0.20241008154146-ea48c09c31ae/go.mod h1:GrIZ12P/9DPOuTpe2PIS51a0P/ZM6iKtByVee1Uf3+k= +go.opentelemetry.io/collector/consumer v0.111.1-0.20241008154146-ea48c09c31ae h1:ps86XqQ6kviggnQ7OnJIHDIoaYYinRGtfKKzcvSuplc= +go.opentelemetry.io/collector/consumer v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:G6pdEDo5A92RY9sBTVbdbuv/nFrVJbDKSO6uRzzgaP8= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.111.1-0.20241008154146-ea48c09c31ae h1:wQHN+NHZ7RwBDy3nSR52LTwLLLVvu116XdXA4KYjqHA= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:GK0QMMiRBWl4IhIF/7ZKgzBlR9SdRSpRlqyNInN4ZoU= +go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae h1:HFj6D19fJYm3KV8QidQmMApmLjzoNkzh8El5OkTGySo= +go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:UDZRrSgaFAwWO6I34fj0KjabVAuBCAnmizsleyIe3I4= +go.opentelemetry.io/collector/internal/globalsignal v0.111.1-0.20241008154146-ea48c09c31ae h1:fublc0EO06p79/OWw2jWVPSPNBMiBcB+0QpLes587DU= +go.opentelemetry.io/collector/internal/globalsignal v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:GqMXodPWOxK5uqpX8MaMXC2389y2XJTa5nPwf8FYDK8= +go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae h1:PcwZe1RD8tC4SZExhf0f5HqK+ZuWGsowHaBBU4PiUv0= +go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae/go.mod h1:Ox1YVLe87cZDB/TL30i4SUz1cA5s6AM6SpFMfY61ICs= +go.opentelemetry.io/collector/pdata/pprofile v0.111.1-0.20241008154146-ea48c09c31ae h1:V6Lp/+A2pei61vmZy8Fwa6j22+wyMQNTFDSe1OVqwWc= +go.opentelemetry.io/collector/pdata/pprofile v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:GRY9BmyYV9myczdT6Q9r+6sK2ICinvzXnrM8+46qpMs= +go.opentelemetry.io/collector/pdata/testdata v0.111.0 h1:Fqyf1NJ0az+HbsvKSCNw8pfa1Y6c4FhZwlMK4ZulG0s= +go.opentelemetry.io/collector/pdata/testdata v0.111.0/go.mod h1:7SypOzbVtRsCkns6Yxa4GztnkVGkk7b9fW24Ow75q5s= +go.opentelemetry.io/collector/pipeline v0.111.1-0.20241008154146-ea48c09c31ae h1:/NNb1rBd/Y42FzIjpLjlRSb7bPANHyI3/3DnPg5p50U= +go.opentelemetry.io/collector/pipeline v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:ZZMU3019geEU283rTW5M/LkcqLqHp/YI2Nl6/Vp68PQ= +go.opentelemetry.io/collector/receiver v0.111.1-0.20241008154146-ea48c09c31ae h1:/CdVIXj9tjTU+5U2D2O/w5T7vYbWF+D6mW9J09GWqis= +go.opentelemetry.io/collector/receiver v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:IB7XEWVIprzSO5Y5RoyPLx3I4ntVGvhOWvoHN1n24IY= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.111.1-0.20241008154146-ea48c09c31ae h1:zJAd63i80I+8wGgK1OB49hX/MJ5GEeS0aNbxxvr7aks= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:cwpkRCGssE2AxydEzkFC3l611d8+csaDH/7BjKC7nHI= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/receiver/systemdreceiver/internal/metadata/generated_status.go b/receiver/systemdreceiver/internal/metadata/generated_status.go new file mode 100644 index 000000000000..e2588d632527 --- /dev/null +++ b/receiver/systemdreceiver/internal/metadata/generated_status.go @@ -0,0 +1,16 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("systemd") + ScopeName = "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/systemdreceiver" +) + +const ( + MetricsStability = component.StabilityLevelDevelopment +) diff --git a/receiver/systemdreceiver/metadata.yaml b/receiver/systemdreceiver/metadata.yaml new file mode 100644 index 000000000000..be0acdfb59bb --- /dev/null +++ b/receiver/systemdreceiver/metadata.yaml @@ -0,0 +1,11 @@ +type: systemd + +status: + class: receiver + stability: + development: [metrics] + distributions: [] + codeowners: + active: [Hemansh31, atoulme] + +tests: diff --git a/receiver/systemdreceiver/receiver.go b/receiver/systemdreceiver/receiver.go new file mode 100644 index 000000000000..ad21a0766210 --- /dev/null +++ b/receiver/systemdreceiver/receiver.go @@ -0,0 +1,20 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package systemdreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/systemdreceiver" +import ( + "context" + + "go.opentelemetry.io/collector/component" +) + +type systemdReceiver struct { +} + +func (s systemdReceiver) Start(_ context.Context, _ component.Host) error { + return nil +} + +func (s systemdReceiver) Shutdown(_ context.Context) error { + return nil +} diff --git a/versions.yaml b/versions.yaml index 30cb52a4201e..9285237ac543 100644 --- a/versions.yaml +++ b/versions.yaml @@ -274,6 +274,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sshcheckreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver + - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/systemdreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tlscheckreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplogreceiver From 694e7cfba2f1b5657e440e6d02f533e5473d2874 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Wed, 16 Oct 2024 22:38:48 -0700 Subject: [PATCH 02/21] [receiver/ntp] Add a new NTP receiver (#35753) #### Description Introduce a new receiver, NTP receiver. This is the first PR as skeleton for the receiver. #### Link to tracking issue Related to #34375 #### Testing N/A #### Documentation metadata.yaml, README. --- .chloggen/ntp.yaml | 27 +++ .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/bug_report.yaml | 1 + .github/ISSUE_TEMPLATE/feature_request.yaml | 1 + .github/ISSUE_TEMPLATE/other.yaml | 1 + .github/ISSUE_TEMPLATE/unmaintained.yaml | 1 + receiver/ntpreceiver/Makefile | 1 + receiver/ntpreceiver/README.md | 45 ++++ receiver/ntpreceiver/config.go | 17 ++ receiver/ntpreceiver/doc.go | 6 + receiver/ntpreceiver/documentation.md | 27 +++ receiver/ntpreceiver/factory.go | 34 +++ .../ntpreceiver/generated_component_test.go | 18 ++ .../ntpreceiver/generated_package_test.go | 13 + receiver/ntpreceiver/go.mod | 55 +++++ receiver/ntpreceiver/go.sum | 136 +++++++++++ .../internal/metadata/generated_config.go | 92 +++++++ .../metadata/generated_config_test.go | 109 +++++++++ .../internal/metadata/generated_metrics.go | 224 ++++++++++++++++++ .../metadata/generated_metrics_test.go | 115 +++++++++ .../internal/metadata/generated_resource.go | 36 +++ .../metadata/generated_resource_test.go | 40 ++++ .../internal/metadata/generated_status.go | 16 ++ .../internal/metadata/testdata/config.yaml | 27 +++ receiver/ntpreceiver/metadata.yaml | 27 +++ versions.yaml | 1 + 26 files changed, 1071 insertions(+) create mode 100644 .chloggen/ntp.yaml create mode 100644 receiver/ntpreceiver/Makefile create mode 100644 receiver/ntpreceiver/README.md create mode 100644 receiver/ntpreceiver/config.go create mode 100644 receiver/ntpreceiver/doc.go create mode 100644 receiver/ntpreceiver/documentation.md create mode 100644 receiver/ntpreceiver/factory.go create mode 100644 receiver/ntpreceiver/generated_component_test.go create mode 100644 receiver/ntpreceiver/generated_package_test.go create mode 100644 receiver/ntpreceiver/go.mod create mode 100644 receiver/ntpreceiver/go.sum create mode 100644 receiver/ntpreceiver/internal/metadata/generated_config.go create mode 100644 receiver/ntpreceiver/internal/metadata/generated_config_test.go create mode 100644 receiver/ntpreceiver/internal/metadata/generated_metrics.go create mode 100644 receiver/ntpreceiver/internal/metadata/generated_metrics_test.go create mode 100644 receiver/ntpreceiver/internal/metadata/generated_resource.go create mode 100644 receiver/ntpreceiver/internal/metadata/generated_resource_test.go create mode 100644 receiver/ntpreceiver/internal/metadata/generated_status.go create mode 100644 receiver/ntpreceiver/internal/metadata/testdata/config.yaml create mode 100644 receiver/ntpreceiver/metadata.yaml diff --git a/.chloggen/ntp.yaml b/.chloggen/ntp.yaml new file mode 100644 index 000000000000..e81621ec443b --- /dev/null +++ b/.chloggen/ntp.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: new_component + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: ntpreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Introduce new receiver reporting the offset between the local machine and a NTP server. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34375] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 37c7aec51903..c40e3e84ba9d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -264,6 +264,7 @@ receiver/mysqlreceiver/ @open-teleme receiver/namedpipereceiver/ @open-telemetry/collector-contrib-approvers @sinkingpoint @djaglowski receiver/nginxreceiver/ @open-telemetry/collector-contrib-approvers @djaglowski receiver/nsxtreceiver/ @open-telemetry/collector-contrib-approvers @dashpole @schmikei +receiver/ntpreceiver/ @open-telemetry/collector-contrib-approvers @atoulme receiver/opencensusreceiver/ @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers receiver/oracledbreceiver/ @open-telemetry/collector-contrib-approvers @dmitryax @crobert-1 @atoulme receiver/osqueryreceiver/ @open-telemetry/collector-contrib-approvers @codeboten @nslaughter @smithclay diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 9d736dadb9f8..04361b20d603 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -259,6 +259,7 @@ body: - receiver/namedpipe - receiver/nginx - receiver/nsxt + - receiver/ntp - receiver/opencensus - receiver/oracledb - receiver/osquery diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 3fd840e735d9..0067eca31d66 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -253,6 +253,7 @@ body: - receiver/namedpipe - receiver/nginx - receiver/nsxt + - receiver/ntp - receiver/opencensus - receiver/oracledb - receiver/osquery diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml index e21e52dbe341..875e32b29e98 100644 --- a/.github/ISSUE_TEMPLATE/other.yaml +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -253,6 +253,7 @@ body: - receiver/namedpipe - receiver/nginx - receiver/nsxt + - receiver/ntp - receiver/opencensus - receiver/oracledb - receiver/osquery diff --git a/.github/ISSUE_TEMPLATE/unmaintained.yaml b/.github/ISSUE_TEMPLATE/unmaintained.yaml index cf6c3dd7dd46..5a86e2567820 100644 --- a/.github/ISSUE_TEMPLATE/unmaintained.yaml +++ b/.github/ISSUE_TEMPLATE/unmaintained.yaml @@ -258,6 +258,7 @@ body: - receiver/namedpipe - receiver/nginx - receiver/nsxt + - receiver/ntp - receiver/opencensus - receiver/oracledb - receiver/osquery diff --git a/receiver/ntpreceiver/Makefile b/receiver/ntpreceiver/Makefile new file mode 100644 index 000000000000..ded7a36092dc --- /dev/null +++ b/receiver/ntpreceiver/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/receiver/ntpreceiver/README.md b/receiver/ntpreceiver/README.md new file mode 100644 index 000000000000..4399e20be527 --- /dev/null +++ b/receiver/ntpreceiver/README.md @@ -0,0 +1,45 @@ +# NTP Receiver + + +| Status | | +| ------------- |-----------| +| Stability | [development]: metrics | +| Distributions | [] | +| Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fntp%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fntp) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fntp%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fntp) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@atoulme](https://www.github.com/atoulme) | + +[development]: https://github.com/open-telemetry/opentelemetry-collector#development + + +> :construction: This receiver is in development. Configuration fields and metric data model are subject to change. + +## Purpose + +This receiver periodically retrieves the clock offset from a NTP server. + +## Configuration + +- `endpoint`: (default = `pool.ntp.org:123`) Endpoint of the NTP server. Must be formatted as `{host}:{port}`. + +- `collection_interval`: (default = `30m`): This receiver collects metrics on an interval. This value must be a string readable by Golang's [time.ParseDuration](https://pkg.go.dev/time#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. + +- `initial_delay` (default = `1s`): defines how long this receiver waits before starting. + +- `metrics` (default: see DefaultMetricsSettings [here])(./internal/metadata/generated_metrics.go): Allows enabling and disabling specific metrics from being collected in this receiver. + +### Example Configuration + +```yaml +receivers: + ntp: + endpoint: pool.ntp.org:123 + collection_interval: 1h + initial_delay: 5m +``` + +The full list of settings exposed for this receiver are documented [here](./config.go) with detailed sample configurations [here](./internal/metadata/testdata/config.yaml). + +## Metrics + +Details about the metrics produced by this receiver can be found in [metadata.yaml](./metadata.yaml) + diff --git a/receiver/ntpreceiver/config.go b/receiver/ntpreceiver/config.go new file mode 100644 index 000000000000..b5f78740c498 --- /dev/null +++ b/receiver/ntpreceiver/config.go @@ -0,0 +1,17 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ntpreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver" + +import ( + "go.opentelemetry.io/collector/receiver/scraperhelper" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver/internal/metadata" +) + +// Config is the configuration for the NSX receiver +type Config struct { + scraperhelper.ControllerConfig `mapstructure:",squash"` + metadata.MetricsBuilderConfig `mapstructure:",squash"` + Endpoint string `mapstructure:"endpoint"` +} diff --git a/receiver/ntpreceiver/doc.go b/receiver/ntpreceiver/doc.go new file mode 100644 index 000000000000..14b5aa92dd4e --- /dev/null +++ b/receiver/ntpreceiver/doc.go @@ -0,0 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mdatagen metadata.yaml + +package ntpreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver" diff --git a/receiver/ntpreceiver/documentation.md b/receiver/ntpreceiver/documentation.md new file mode 100644 index 000000000000..182193963883 --- /dev/null +++ b/receiver/ntpreceiver/documentation.md @@ -0,0 +1,27 @@ +[comment]: <> (Code generated by mdatagen. DO NOT EDIT.) + +# ntp + +## Default Metrics + +The following metrics are emitted by default. Each of them can be disabled by applying the following configuration: + +```yaml +metrics: + : + enabled: false +``` + +### ntp.offset + +Time difference between local and NTP server clocks in seconds. + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| s | Gauge | Int | + +## Resource Attributes + +| Name | Description | Values | Enabled | +| ---- | ----------- | ------ | ------- | +| ntp.host | NTP server used. Corresponds to configured `host`. | Any Str | true | diff --git a/receiver/ntpreceiver/factory.go b/receiver/ntpreceiver/factory.go new file mode 100644 index 000000000000..07b3a9a7534f --- /dev/null +++ b/receiver/ntpreceiver/factory.go @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ntpreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver" + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/scraperhelper" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver/internal/metadata" +) + +func NewFactory() receiver.Factory { + return receiver.NewFactory( + metadata.Type, + createDefaultConfig, + receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability), + ) +} + +func createDefaultConfig() component.Config { + return &Config{ + ControllerConfig: scraperhelper.NewDefaultControllerConfig(), + MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(), + } +} + +func createMetricsReceiver(_ context.Context, _ receiver.Settings, _ component.Config, _ consumer.Metrics) (receiver.Metrics, error) { + return nil, nil +} diff --git a/receiver/ntpreceiver/generated_component_test.go b/receiver/ntpreceiver/generated_component_test.go new file mode 100644 index 000000000000..5e8b22347941 --- /dev/null +++ b/receiver/ntpreceiver/generated_component_test.go @@ -0,0 +1,18 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package ntpreceiver + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" +) + +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, "ntp", NewFactory().Type().String()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} diff --git a/receiver/ntpreceiver/generated_package_test.go b/receiver/ntpreceiver/generated_package_test.go new file mode 100644 index 000000000000..02b7cfbd4e87 --- /dev/null +++ b/receiver/ntpreceiver/generated_package_test.go @@ -0,0 +1,13 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package ntpreceiver + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/receiver/ntpreceiver/go.mod b/receiver/ntpreceiver/go.mod new file mode 100644 index 000000000000..71d67ef93dd2 --- /dev/null +++ b/receiver/ntpreceiver/go.mod @@ -0,0 +1,55 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver + +go 1.22.0 + +require ( + github.com/google/go-cmp v0.6.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/confmap v1.17.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/consumer v0.111.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/filter v0.111.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/receiver v0.111.1-0.20241008154146-ea48c09c31ae + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/consumer/consumerprofiles v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/internal/globalsignal v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/pipeline v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/collector/receiver/receiverprofiles v0.111.1-0.20241008154146-ea48c09c31ae // indirect + go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/receiver/ntpreceiver/go.sum b/receiver/ntpreceiver/go.sum new file mode 100644 index 000000000000..65f0a94a89a6 --- /dev/null +++ b/receiver/ntpreceiver/go.sum @@ -0,0 +1,136 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= +github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae h1:dXAMqXGJp1vWG7qwS/2sjIyJgmyOSfEOm6Gcmkzp1cQ= +go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:iWUfPxpVwZhkI4v3/Gh5wt4iKyJn4lriPFAug8iLXno= +go.opentelemetry.io/collector/config/configtelemetry v0.111.1-0.20241008154146-ea48c09c31ae h1:NmNYRBSP+IUK9CsU1Q/1eS/tXmYTPMYxmGQsxEprq/s= +go.opentelemetry.io/collector/config/configtelemetry v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:R0MBUxjSMVMIhljuDHWIygzzJWQyZHXXWIgQNxcFwhc= +go.opentelemetry.io/collector/confmap v1.17.1-0.20241008154146-ea48c09c31ae h1:mYgomPNE0dq1SU1OVnMb/Z8Xbj89jBXnjJexz9M71t0= +go.opentelemetry.io/collector/confmap v1.17.1-0.20241008154146-ea48c09c31ae/go.mod h1:GrIZ12P/9DPOuTpe2PIS51a0P/ZM6iKtByVee1Uf3+k= +go.opentelemetry.io/collector/consumer v0.111.1-0.20241008154146-ea48c09c31ae h1:ps86XqQ6kviggnQ7OnJIHDIoaYYinRGtfKKzcvSuplc= +go.opentelemetry.io/collector/consumer v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:G6pdEDo5A92RY9sBTVbdbuv/nFrVJbDKSO6uRzzgaP8= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.111.1-0.20241008154146-ea48c09c31ae h1:wQHN+NHZ7RwBDy3nSR52LTwLLLVvu116XdXA4KYjqHA= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:GK0QMMiRBWl4IhIF/7ZKgzBlR9SdRSpRlqyNInN4ZoU= +go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae h1:HFj6D19fJYm3KV8QidQmMApmLjzoNkzh8El5OkTGySo= +go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:UDZRrSgaFAwWO6I34fj0KjabVAuBCAnmizsleyIe3I4= +go.opentelemetry.io/collector/filter v0.111.1-0.20241008154146-ea48c09c31ae h1:fLRV9bU33PJWQ/eZCwzfKdV0I9ljhP84Zoq9+tBhcLU= +go.opentelemetry.io/collector/filter v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:74Acew42eexKiuLu3tVehyMK4b5XJPWXoJyNjK2FM+U= +go.opentelemetry.io/collector/internal/globalsignal v0.111.1-0.20241008154146-ea48c09c31ae h1:fublc0EO06p79/OWw2jWVPSPNBMiBcB+0QpLes587DU= +go.opentelemetry.io/collector/internal/globalsignal v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:GqMXodPWOxK5uqpX8MaMXC2389y2XJTa5nPwf8FYDK8= +go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae h1:PcwZe1RD8tC4SZExhf0f5HqK+ZuWGsowHaBBU4PiUv0= +go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae/go.mod h1:Ox1YVLe87cZDB/TL30i4SUz1cA5s6AM6SpFMfY61ICs= +go.opentelemetry.io/collector/pdata/pprofile v0.111.1-0.20241008154146-ea48c09c31ae h1:V6Lp/+A2pei61vmZy8Fwa6j22+wyMQNTFDSe1OVqwWc= +go.opentelemetry.io/collector/pdata/pprofile v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:GRY9BmyYV9myczdT6Q9r+6sK2ICinvzXnrM8+46qpMs= +go.opentelemetry.io/collector/pdata/testdata v0.111.0 h1:Fqyf1NJ0az+HbsvKSCNw8pfa1Y6c4FhZwlMK4ZulG0s= +go.opentelemetry.io/collector/pdata/testdata v0.111.0/go.mod h1:7SypOzbVtRsCkns6Yxa4GztnkVGkk7b9fW24Ow75q5s= +go.opentelemetry.io/collector/pipeline v0.111.1-0.20241008154146-ea48c09c31ae h1:/NNb1rBd/Y42FzIjpLjlRSb7bPANHyI3/3DnPg5p50U= +go.opentelemetry.io/collector/pipeline v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:ZZMU3019geEU283rTW5M/LkcqLqHp/YI2Nl6/Vp68PQ= +go.opentelemetry.io/collector/receiver v0.111.1-0.20241008154146-ea48c09c31ae h1:/CdVIXj9tjTU+5U2D2O/w5T7vYbWF+D6mW9J09GWqis= +go.opentelemetry.io/collector/receiver v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:IB7XEWVIprzSO5Y5RoyPLx3I4ntVGvhOWvoHN1n24IY= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.111.1-0.20241008154146-ea48c09c31ae h1:zJAd63i80I+8wGgK1OB49hX/MJ5GEeS0aNbxxvr7aks= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.111.1-0.20241008154146-ea48c09c31ae/go.mod h1:cwpkRCGssE2AxydEzkFC3l611d8+csaDH/7BjKC7nHI= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= +go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/receiver/ntpreceiver/internal/metadata/generated_config.go b/receiver/ntpreceiver/internal/metadata/generated_config.go new file mode 100644 index 000000000000..1e2e1d127fed --- /dev/null +++ b/receiver/ntpreceiver/internal/metadata/generated_config.go @@ -0,0 +1,92 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/filter" +) + +// MetricConfig provides common config for a particular metric. +type MetricConfig struct { + Enabled bool `mapstructure:"enabled"` + + enabledSetByUser bool +} + +func (ms *MetricConfig) Unmarshal(parser *confmap.Conf) error { + if parser == nil { + return nil + } + err := parser.Unmarshal(ms) + if err != nil { + return err + } + ms.enabledSetByUser = parser.IsSet("enabled") + return nil +} + +// MetricsConfig provides config for ntp metrics. +type MetricsConfig struct { + NtpOffset MetricConfig `mapstructure:"ntp.offset"` +} + +func DefaultMetricsConfig() MetricsConfig { + return MetricsConfig{ + NtpOffset: MetricConfig{ + Enabled: true, + }, + } +} + +// ResourceAttributeConfig provides common config for a particular resource attribute. +type ResourceAttributeConfig struct { + Enabled bool `mapstructure:"enabled"` + // Experimental: MetricsInclude defines a list of filters for attribute values. + // If the list is not empty, only metrics with matching resource attribute values will be emitted. + MetricsInclude []filter.Config `mapstructure:"metrics_include"` + // Experimental: MetricsExclude defines a list of filters for attribute values. + // If the list is not empty, metrics with matching resource attribute values will not be emitted. + // MetricsInclude has higher priority than MetricsExclude. + MetricsExclude []filter.Config `mapstructure:"metrics_exclude"` + + enabledSetByUser bool +} + +func (rac *ResourceAttributeConfig) Unmarshal(parser *confmap.Conf) error { + if parser == nil { + return nil + } + err := parser.Unmarshal(rac) + if err != nil { + return err + } + rac.enabledSetByUser = parser.IsSet("enabled") + return nil +} + +// ResourceAttributesConfig provides config for ntp resource attributes. +type ResourceAttributesConfig struct { + NtpHost ResourceAttributeConfig `mapstructure:"ntp.host"` +} + +func DefaultResourceAttributesConfig() ResourceAttributesConfig { + return ResourceAttributesConfig{ + NtpHost: ResourceAttributeConfig{ + Enabled: true, + }, + } +} + +// MetricsBuilderConfig is a configuration for ntp metrics builder. +type MetricsBuilderConfig struct { + Metrics MetricsConfig `mapstructure:"metrics"` + ResourceAttributes ResourceAttributesConfig `mapstructure:"resource_attributes"` +} + +func DefaultMetricsBuilderConfig() MetricsBuilderConfig { + return MetricsBuilderConfig{ + Metrics: DefaultMetricsConfig(), + ResourceAttributes: DefaultResourceAttributesConfig(), + } +} diff --git a/receiver/ntpreceiver/internal/metadata/generated_config_test.go b/receiver/ntpreceiver/internal/metadata/generated_config_test.go new file mode 100644 index 000000000000..5653b983743e --- /dev/null +++ b/receiver/ntpreceiver/internal/metadata/generated_config_test.go @@ -0,0 +1,109 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap/confmaptest" +) + +func TestMetricsBuilderConfig(t *testing.T) { + tests := []struct { + name string + want MetricsBuilderConfig + }{ + { + name: "default", + want: DefaultMetricsBuilderConfig(), + }, + { + name: "all_set", + want: MetricsBuilderConfig{ + Metrics: MetricsConfig{ + NtpOffset: MetricConfig{Enabled: true}, + }, + ResourceAttributes: ResourceAttributesConfig{ + NtpHost: ResourceAttributeConfig{Enabled: true}, + }, + }, + }, + { + name: "none_set", + want: MetricsBuilderConfig{ + Metrics: MetricsConfig{ + NtpOffset: MetricConfig{Enabled: false}, + }, + ResourceAttributes: ResourceAttributesConfig{ + NtpHost: ResourceAttributeConfig{Enabled: false}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := loadMetricsBuilderConfig(t, tt.name) + if diff := cmp.Diff(tt.want, cfg, cmpopts.IgnoreUnexported(MetricConfig{}, ResourceAttributeConfig{})); diff != "" { + t.Errorf("Config mismatch (-expected +actual):\n%s", diff) + } + }) + } +} + +func loadMetricsBuilderConfig(t *testing.T, name string) MetricsBuilderConfig { + cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) + require.NoError(t, err) + sub, err := cm.Sub(name) + require.NoError(t, err) + cfg := DefaultMetricsBuilderConfig() + require.NoError(t, sub.Unmarshal(&cfg)) + return cfg +} + +func TestResourceAttributesConfig(t *testing.T) { + tests := []struct { + name string + want ResourceAttributesConfig + }{ + { + name: "default", + want: DefaultResourceAttributesConfig(), + }, + { + name: "all_set", + want: ResourceAttributesConfig{ + NtpHost: ResourceAttributeConfig{Enabled: true}, + }, + }, + { + name: "none_set", + want: ResourceAttributesConfig{ + NtpHost: ResourceAttributeConfig{Enabled: false}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := loadResourceAttributesConfig(t, tt.name) + if diff := cmp.Diff(tt.want, cfg, cmpopts.IgnoreUnexported(ResourceAttributeConfig{})); diff != "" { + t.Errorf("Config mismatch (-expected +actual):\n%s", diff) + } + }) + } +} + +func loadResourceAttributesConfig(t *testing.T, name string) ResourceAttributesConfig { + cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) + require.NoError(t, err) + sub, err := cm.Sub(name) + require.NoError(t, err) + sub, err = sub.Sub("resource_attributes") + require.NoError(t, err) + cfg := DefaultResourceAttributesConfig() + require.NoError(t, sub.Unmarshal(&cfg)) + return cfg +} diff --git a/receiver/ntpreceiver/internal/metadata/generated_metrics.go b/receiver/ntpreceiver/internal/metadata/generated_metrics.go new file mode 100644 index 000000000000..afc023c251d6 --- /dev/null +++ b/receiver/ntpreceiver/internal/metadata/generated_metrics.go @@ -0,0 +1,224 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/filter" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/receiver" +) + +type metricNtpOffset struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills ntp.offset metric with initial data. +func (m *metricNtpOffset) init() { + m.data.SetName("ntp.offset") + m.data.SetDescription("Time difference between local and NTP server clocks in seconds.") + m.data.SetUnit("s") + m.data.SetEmptyGauge() +} + +func (m *metricNtpOffset) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricNtpOffset) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricNtpOffset) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricNtpOffset(cfg MetricConfig) metricNtpOffset { + m := metricNtpOffset{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +// MetricsBuilder provides an interface for scrapers to report metrics while taking care of all the transformations +// required to produce metric representation defined in metadata and user config. +type MetricsBuilder struct { + config MetricsBuilderConfig // config of the metrics builder. + startTime pcommon.Timestamp // start time that will be applied to all recorded data points. + metricsCapacity int // maximum observed number of metrics per resource. + metricsBuffer pmetric.Metrics // accumulates metrics data before emitting. + buildInfo component.BuildInfo // contains version information. + resourceAttributeIncludeFilter map[string]filter.Filter + resourceAttributeExcludeFilter map[string]filter.Filter + metricNtpOffset metricNtpOffset +} + +// MetricBuilderOption applies changes to default metrics builder. +type MetricBuilderOption interface { + apply(*MetricsBuilder) +} + +type metricBuilderOptionFunc func(mb *MetricsBuilder) + +func (mbof metricBuilderOptionFunc) apply(mb *MetricsBuilder) { + mbof(mb) +} + +// WithStartTime sets startTime on the metrics builder. +func WithStartTime(startTime pcommon.Timestamp) MetricBuilderOption { + return metricBuilderOptionFunc(func(mb *MetricsBuilder) { + mb.startTime = startTime + }) +} + +func NewMetricsBuilder(mbc MetricsBuilderConfig, settings receiver.Settings, options ...MetricBuilderOption) *MetricsBuilder { + mb := &MetricsBuilder{ + config: mbc, + startTime: pcommon.NewTimestampFromTime(time.Now()), + metricsBuffer: pmetric.NewMetrics(), + buildInfo: settings.BuildInfo, + metricNtpOffset: newMetricNtpOffset(mbc.Metrics.NtpOffset), + resourceAttributeIncludeFilter: make(map[string]filter.Filter), + resourceAttributeExcludeFilter: make(map[string]filter.Filter), + } + if mbc.ResourceAttributes.NtpHost.MetricsInclude != nil { + mb.resourceAttributeIncludeFilter["ntp.host"] = filter.CreateFilter(mbc.ResourceAttributes.NtpHost.MetricsInclude) + } + if mbc.ResourceAttributes.NtpHost.MetricsExclude != nil { + mb.resourceAttributeExcludeFilter["ntp.host"] = filter.CreateFilter(mbc.ResourceAttributes.NtpHost.MetricsExclude) + } + + for _, op := range options { + op.apply(mb) + } + return mb +} + +// NewResourceBuilder returns a new resource builder that should be used to build a resource associated with for the emitted metrics. +func (mb *MetricsBuilder) NewResourceBuilder() *ResourceBuilder { + return NewResourceBuilder(mb.config.ResourceAttributes) +} + +// updateCapacity updates max length of metrics and resource attributes that will be used for the slice capacity. +func (mb *MetricsBuilder) updateCapacity(rm pmetric.ResourceMetrics) { + if mb.metricsCapacity < rm.ScopeMetrics().At(0).Metrics().Len() { + mb.metricsCapacity = rm.ScopeMetrics().At(0).Metrics().Len() + } +} + +// ResourceMetricsOption applies changes to provided resource metrics. +type ResourceMetricsOption interface { + apply(pmetric.ResourceMetrics) +} + +type resourceMetricsOptionFunc func(pmetric.ResourceMetrics) + +func (rmof resourceMetricsOptionFunc) apply(rm pmetric.ResourceMetrics) { + rmof(rm) +} + +// WithResource sets the provided resource on the emitted ResourceMetrics. +// It's recommended to use ResourceBuilder to create the resource. +func WithResource(res pcommon.Resource) ResourceMetricsOption { + return resourceMetricsOptionFunc(func(rm pmetric.ResourceMetrics) { + res.CopyTo(rm.Resource()) + }) +} + +// WithStartTimeOverride overrides start time for all the resource metrics data points. +// This option should be only used if different start time has to be set on metrics coming from different resources. +func WithStartTimeOverride(start pcommon.Timestamp) ResourceMetricsOption { + return resourceMetricsOptionFunc(func(rm pmetric.ResourceMetrics) { + var dps pmetric.NumberDataPointSlice + metrics := rm.ScopeMetrics().At(0).Metrics() + for i := 0; i < metrics.Len(); i++ { + switch metrics.At(i).Type() { + case pmetric.MetricTypeGauge: + dps = metrics.At(i).Gauge().DataPoints() + case pmetric.MetricTypeSum: + dps = metrics.At(i).Sum().DataPoints() + } + for j := 0; j < dps.Len(); j++ { + dps.At(j).SetStartTimestamp(start) + } + } + }) +} + +// EmitForResource saves all the generated metrics under a new resource and updates the internal state to be ready for +// recording another set of data points as part of another resource. This function can be helpful when one scraper +// needs to emit metrics from several resources. Otherwise calling this function is not required, +// just `Emit` function can be called instead. +// Resource attributes should be provided as ResourceMetricsOption arguments. +func (mb *MetricsBuilder) EmitForResource(options ...ResourceMetricsOption) { + rm := pmetric.NewResourceMetrics() + ils := rm.ScopeMetrics().AppendEmpty() + ils.Scope().SetName("github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver") + ils.Scope().SetVersion(mb.buildInfo.Version) + ils.Metrics().EnsureCapacity(mb.metricsCapacity) + mb.metricNtpOffset.emit(ils.Metrics()) + + for _, op := range options { + op.apply(rm) + } + for attr, filter := range mb.resourceAttributeIncludeFilter { + if val, ok := rm.Resource().Attributes().Get(attr); ok && !filter.Matches(val.AsString()) { + return + } + } + for attr, filter := range mb.resourceAttributeExcludeFilter { + if val, ok := rm.Resource().Attributes().Get(attr); ok && filter.Matches(val.AsString()) { + return + } + } + + if ils.Metrics().Len() > 0 { + mb.updateCapacity(rm) + rm.MoveTo(mb.metricsBuffer.ResourceMetrics().AppendEmpty()) + } +} + +// Emit returns all the metrics accumulated by the metrics builder and updates the internal state to be ready for +// recording another set of metrics. This function will be responsible for applying all the transformations required to +// produce metric representation defined in metadata and user config, e.g. delta or cumulative. +func (mb *MetricsBuilder) Emit(options ...ResourceMetricsOption) pmetric.Metrics { + mb.EmitForResource(options...) + metrics := mb.metricsBuffer + mb.metricsBuffer = pmetric.NewMetrics() + return metrics +} + +// RecordNtpOffsetDataPoint adds a data point to ntp.offset metric. +func (mb *MetricsBuilder) RecordNtpOffsetDataPoint(ts pcommon.Timestamp, val int64) { + mb.metricNtpOffset.recordDataPoint(mb.startTime, ts, val) +} + +// Reset resets metrics builder to its initial state. It should be used when external metrics source is restarted, +// and metrics builder should update its startTime and reset it's internal state accordingly. +func (mb *MetricsBuilder) Reset(options ...MetricBuilderOption) { + mb.startTime = pcommon.NewTimestampFromTime(time.Now()) + for _, op := range options { + op.apply(mb) + } +} diff --git a/receiver/ntpreceiver/internal/metadata/generated_metrics_test.go b/receiver/ntpreceiver/internal/metadata/generated_metrics_test.go new file mode 100644 index 000000000000..4ea82d6f71ac --- /dev/null +++ b/receiver/ntpreceiver/internal/metadata/generated_metrics_test.go @@ -0,0 +1,115 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/receiver/receivertest" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +type testDataSet int + +const ( + testDataSetDefault testDataSet = iota + testDataSetAll + testDataSetNone +) + +func TestMetricsBuilder(t *testing.T) { + tests := []struct { + name string + metricsSet testDataSet + resAttrsSet testDataSet + expectEmpty bool + }{ + { + name: "default", + }, + { + name: "all_set", + metricsSet: testDataSetAll, + resAttrsSet: testDataSetAll, + }, + { + name: "none_set", + metricsSet: testDataSetNone, + resAttrsSet: testDataSetNone, + expectEmpty: true, + }, + { + name: "filter_set_include", + resAttrsSet: testDataSetAll, + }, + { + name: "filter_set_exclude", + resAttrsSet: testDataSetAll, + expectEmpty: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + start := pcommon.Timestamp(1_000_000_000) + ts := pcommon.Timestamp(1_000_001_000) + observedZapCore, observedLogs := observer.New(zap.WarnLevel) + settings := receivertest.NewNopSettings() + settings.Logger = zap.New(observedZapCore) + mb := NewMetricsBuilder(loadMetricsBuilderConfig(t, tt.name), settings, WithStartTime(start)) + + expectedWarnings := 0 + + assert.Equal(t, expectedWarnings, observedLogs.Len()) + + defaultMetricsCount := 0 + allMetricsCount := 0 + + defaultMetricsCount++ + allMetricsCount++ + mb.RecordNtpOffsetDataPoint(ts, 1) + + rb := mb.NewResourceBuilder() + rb.SetNtpHost("ntp.host-val") + res := rb.Emit() + metrics := mb.Emit(WithResource(res)) + + if tt.expectEmpty { + assert.Equal(t, 0, metrics.ResourceMetrics().Len()) + return + } + + assert.Equal(t, 1, metrics.ResourceMetrics().Len()) + rm := metrics.ResourceMetrics().At(0) + assert.Equal(t, res, rm.Resource()) + assert.Equal(t, 1, rm.ScopeMetrics().Len()) + ms := rm.ScopeMetrics().At(0).Metrics() + if tt.metricsSet == testDataSetDefault { + assert.Equal(t, defaultMetricsCount, ms.Len()) + } + if tt.metricsSet == testDataSetAll { + assert.Equal(t, allMetricsCount, ms.Len()) + } + validatedMetrics := make(map[string]bool) + for i := 0; i < ms.Len(); i++ { + switch ms.At(i).Name() { + case "ntp.offset": + assert.False(t, validatedMetrics["ntp.offset"], "Found a duplicate in the metrics slice: ntp.offset") + validatedMetrics["ntp.offset"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "Time difference between local and NTP server clocks in seconds.", ms.At(i).Description()) + assert.Equal(t, "s", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + } + } + }) + } +} diff --git a/receiver/ntpreceiver/internal/metadata/generated_resource.go b/receiver/ntpreceiver/internal/metadata/generated_resource.go new file mode 100644 index 000000000000..d857b849d334 --- /dev/null +++ b/receiver/ntpreceiver/internal/metadata/generated_resource.go @@ -0,0 +1,36 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" +) + +// ResourceBuilder is a helper struct to build resources predefined in metadata.yaml. +// The ResourceBuilder is not thread-safe and must not to be used in multiple goroutines. +type ResourceBuilder struct { + config ResourceAttributesConfig + res pcommon.Resource +} + +// NewResourceBuilder creates a new ResourceBuilder. This method should be called on the start of the application. +func NewResourceBuilder(rac ResourceAttributesConfig) *ResourceBuilder { + return &ResourceBuilder{ + config: rac, + res: pcommon.NewResource(), + } +} + +// SetNtpHost sets provided value as "ntp.host" attribute. +func (rb *ResourceBuilder) SetNtpHost(val string) { + if rb.config.NtpHost.Enabled { + rb.res.Attributes().PutStr("ntp.host", val) + } +} + +// Emit returns the built resource and resets the internal builder state. +func (rb *ResourceBuilder) Emit() pcommon.Resource { + r := rb.res + rb.res = pcommon.NewResource() + return r +} diff --git a/receiver/ntpreceiver/internal/metadata/generated_resource_test.go b/receiver/ntpreceiver/internal/metadata/generated_resource_test.go new file mode 100644 index 000000000000..3a1e5638dc83 --- /dev/null +++ b/receiver/ntpreceiver/internal/metadata/generated_resource_test.go @@ -0,0 +1,40 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResourceBuilder(t *testing.T) { + for _, tt := range []string{"default", "all_set", "none_set"} { + t.Run(tt, func(t *testing.T) { + cfg := loadResourceAttributesConfig(t, tt) + rb := NewResourceBuilder(cfg) + rb.SetNtpHost("ntp.host-val") + + res := rb.Emit() + assert.Equal(t, 0, rb.Emit().Attributes().Len()) // Second call should return empty Resource + + switch tt { + case "default": + assert.Equal(t, 1, res.Attributes().Len()) + case "all_set": + assert.Equal(t, 1, res.Attributes().Len()) + case "none_set": + assert.Equal(t, 0, res.Attributes().Len()) + return + default: + assert.Failf(t, "unexpected test case: %s", tt) + } + + val, ok := res.Attributes().Get("ntp.host") + assert.True(t, ok) + if ok { + assert.EqualValues(t, "ntp.host-val", val.Str()) + } + }) + } +} diff --git a/receiver/ntpreceiver/internal/metadata/generated_status.go b/receiver/ntpreceiver/internal/metadata/generated_status.go new file mode 100644 index 000000000000..9b566901d750 --- /dev/null +++ b/receiver/ntpreceiver/internal/metadata/generated_status.go @@ -0,0 +1,16 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("ntp") + ScopeName = "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver" +) + +const ( + MetricsStability = component.StabilityLevelDevelopment +) diff --git a/receiver/ntpreceiver/internal/metadata/testdata/config.yaml b/receiver/ntpreceiver/internal/metadata/testdata/config.yaml new file mode 100644 index 000000000000..d72166d842fb --- /dev/null +++ b/receiver/ntpreceiver/internal/metadata/testdata/config.yaml @@ -0,0 +1,27 @@ +default: +all_set: + metrics: + ntp.offset: + enabled: true + resource_attributes: + ntp.host: + enabled: true +none_set: + metrics: + ntp.offset: + enabled: false + resource_attributes: + ntp.host: + enabled: false +filter_set_include: + resource_attributes: + ntp.host: + enabled: true + metrics_include: + - regexp: ".*" +filter_set_exclude: + resource_attributes: + ntp.host: + enabled: true + metrics_exclude: + - strict: "ntp.host-val" diff --git a/receiver/ntpreceiver/metadata.yaml b/receiver/ntpreceiver/metadata.yaml new file mode 100644 index 000000000000..c6536b0a05c9 --- /dev/null +++ b/receiver/ntpreceiver/metadata.yaml @@ -0,0 +1,27 @@ +type: ntp + +status: + class: receiver + stability: + development: [metrics] + distributions: [] + codeowners: + active: [atoulme] + +resource_attributes: + ntp.host: + description: NTP server used. Corresponds to configured `host`. + enabled: true + type: string + +metrics: + ntp.offset: + description: Time difference between local and NTP server clocks in seconds. + unit: "s" + gauge: + value_type: int + enabled: true + +tests: + skip_lifecycle: true + skip_shutdown: true \ No newline at end of file diff --git a/versions.yaml b/versions.yaml index 9285237ac543..95dd67608171 100644 --- a/versions.yaml +++ b/versions.yaml @@ -242,6 +242,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/namedpipereceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/nginxreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/nsxtreceiver + - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/opencensusreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/oracledbreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/osqueryreceiver From e7ebc6e1676aa661880a09b0ff93a9cccad8f011 Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Thu, 17 Oct 2024 05:47:36 -0300 Subject: [PATCH 03/21] [exporter/elasticsearch] Add mapping mode bodymap (#35637) #### Description This PR implements a new mapping mode `bodymap` that works by serializing each LogRecord body as-is into a separate document for ingestion. Fixes #35444 #### Testing #### Documentation --------- Co-authored-by: Carson Ip --- ...feature_elasticsearch_mapping_bodymap.yaml | 27 ++++ exporter/elasticsearchexporter/README.md | 9 +- exporter/elasticsearchexporter/config.go | 4 + exporter/elasticsearchexporter/exporter.go | 5 + .../elasticsearchexporter/exporter_test.go | 125 ++++++++++++++++++ exporter/elasticsearchexporter/go.mod | 2 +- exporter/elasticsearchexporter/model.go | 14 ++ exporter/elasticsearchexporter/model_test.go | 40 ++++++ 8 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 .chloggen/feature_elasticsearch_mapping_bodymap.yaml diff --git a/.chloggen/feature_elasticsearch_mapping_bodymap.yaml b/.chloggen/feature_elasticsearch_mapping_bodymap.yaml new file mode 100644 index 000000000000..9d749c7b3c45 --- /dev/null +++ b/.chloggen/feature_elasticsearch_mapping_bodymap.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Introduce an experimental bodymap mapping mode for logs + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35444] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] \ No newline at end of file diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 13564bbd012a..5ec203f13674 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -155,14 +155,19 @@ behaviours, which may be configured through the following settings: - `none`: Use original fields and event structure from the OTLP event. - `ecs`: Try to map fields to [Elastic Common Schema (ECS)][ECS] - `otel`: Elastic's preferred "OTel-native" mapping mode. Uses original fields and event structure from the OTLP event. - - :warning: This mode's behavior is unstable, it is currently is experimental and undergoing changes. + - :warning: This mode's behavior is unstable, it is currently experimental and undergoing changes. - There's a special treatment for the following attributes: `data_stream.type`, `data_stream.dataset`, `data_stream.namespace`. Instead of serializing these values under the `*attributes.*` namespace, they're put at the root of the document, to conform with the conventions of the data stream naming scheme that maps these as `constant_keyword` fields. - `data_stream.dataset` will always be appended with `.otel`. It is recommended to use with `*_dynamic_index.enabled: true` to route documents to data stream `${data_stream.type}-${data_stream.dataset}-${data_stream.namespace}`. - Span events are stored in separate documents. They will be routed with `data_stream.type` set to `logs` if `traces_dynamic_index::enabled` is `true`. - `raw`: Omit the `Attributes.` string prefixed to field names for log and span attributes as well as omit the `Events.` string prefixed to - field names for span events. + field names for span events. + - `bodymap`: Provides fine-grained control over the final documents to be ingested. + :warning: This mode's behavior is unstable, it is currently experimental and undergoing changes. + It works only for logs where the log record body is a map. Each LogRecord + body is serialized to JSON as-is and becomes a separate document for ingestion. + If the log record body is not a map, the exporter will log a warning and drop the log record. - `dedup` (DEPRECATED). This configuration is deprecated and non-operational, and will be removed in the future. Object keys are always deduplicated to avoid Elasticsearch rejecting documents. diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 547ff693894a..072bd725c6fe 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -207,6 +207,7 @@ const ( MappingECS MappingOTel MappingRaw + MappingBodyMap ) var ( @@ -224,6 +225,8 @@ func (m MappingMode) String() string { return "otel" case MappingRaw: return "raw" + case MappingBodyMap: + return "bodymap" default: return "" } @@ -236,6 +239,7 @@ var mappingModes = func() map[string]MappingMode { MappingECS, MappingOTel, MappingRaw, + MappingBodyMap, } { table[strings.ToLower(m.String())] = m } diff --git a/exporter/elasticsearchexporter/exporter.go b/exporter/elasticsearchexporter/exporter.go index b7e00b713cd8..ebd3800858a2 100644 --- a/exporter/elasticsearchexporter/exporter.go +++ b/exporter/elasticsearchexporter/exporter.go @@ -131,6 +131,11 @@ func (e *elasticsearchExporter) pushLogsData(ctx context.Context, ld plog.Logs) return cerr } + if errors.Is(err, ErrInvalidTypeForBodyMapMode) { + e.Logger.Warn("dropping log record", zap.Error(err)) + continue + } + errs = append(errs, err) } } diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index 13de09564d32..e2871666b138 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -81,6 +81,131 @@ func TestExporterLogs(t *testing.T) { rec.WaitItems(1) }) + t.Run("publish with bodymap encoding", func(t *testing.T) { + tableTests := []struct { + name string + body func() pcommon.Value + expected string + }{ + { + name: "flat", + body: func() pcommon.Value { + body := pcommon.NewValueMap() + m := body.Map() + m.PutStr("@timestamp", "2024-03-12T20:00:41.123456789Z") + m.PutInt("id", 1) + m.PutStr("key", "value") + return body + }, + expected: `{"@timestamp":"2024-03-12T20:00:41.123456789Z","id":1,"key":"value"}`, + }, + { + name: "dotted key", + body: func() pcommon.Value { + body := pcommon.NewValueMap() + m := body.Map() + m.PutInt("a", 1) + m.PutInt("a.b", 2) + m.PutInt("a.b.c", 3) + return body + }, + expected: `{"a":1,"a.b":2,"a.b.c":3}`, + }, + { + name: "slice", + body: func() pcommon.Value { + body := pcommon.NewValueMap() + m := body.Map() + s := m.PutEmptySlice("a") + for i := 0; i < 2; i++ { + s.AppendEmpty().SetInt(int64(i)) + } + return body + }, + expected: `{"a":[0,1]}`, + }, + { + name: "inner map", + body: func() pcommon.Value { + body := pcommon.NewValueMap() + m := body.Map() + m1 := m.PutEmptyMap("a") + m1.PutInt("b", 1) + m1.PutInt("c", 2) + return body + }, + expected: `{"a":{"b":1,"c":2}}`, + }, + { + name: "nested map", + body: func() pcommon.Value { + body := pcommon.NewValueMap() + m := body.Map() + m1 := m.PutEmptyMap("a") + m2 := m1.PutEmptyMap("b") + m2.PutInt("c", 1) + m2.PutInt("d", 2) + return body + }, + expected: `{"a":{"b":{"c":1,"d":2}}}`, + }, + } + + for _, tt := range tableTests { + t.Run(tt.name, func(t *testing.T) { + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + assert.JSONEq(t, tt.expected, string(docs[0].Document)) + return itemsAllOK(docs) + }) + + exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { + cfg.Mapping.Mode = "bodymap" + }) + logs := plog.NewLogs() + resourceLogs := logs.ResourceLogs().AppendEmpty() + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + logRecords := scopeLogs.LogRecords() + logRecord := logRecords.AppendEmpty() + tt.body().CopyTo(logRecord.Body()) + + mustSendLogs(t, exporter, logs) + rec.WaitItems(1) + }) + } + }) + + t.Run("drops log records for bodymap mode if body is not a map", func(t *testing.T) { + logs := plog.NewLogs() + resourceLogs := logs.ResourceLogs().AppendEmpty() + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + logRecords := scopeLogs.LogRecords() + + // Invalid body type should be dropped. + logRecords.AppendEmpty().Body().SetEmptySlice() + + // We should still process the valid records in the batch. + bodyMap := logRecords.AppendEmpty().Body().SetEmptyMap() + bodyMap.PutInt("a", 42) + + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + defer rec.Record(docs) + assert.Len(t, docs, 1) + assert.JSONEq(t, `{"a":42}`, string(docs[0].Document)) + return itemsAllOK(docs) + }) + + exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { + cfg.Mapping.Mode = "bodymap" + }) + + err := exporter.ConsumeLogs(context.Background(), logs) + assert.NoError(t, err) + rec.WaitItems(1) + }) + t.Run("publish with dedot", func(t *testing.T) { rec := newBulkRecorder() server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { diff --git a/exporter/elasticsearchexporter/go.mod b/exporter/elasticsearchexporter/go.mod index 3f1046727987..c9a675710622 100644 --- a/exporter/elasticsearchexporter/go.mod +++ b/exporter/elasticsearchexporter/go.mod @@ -7,6 +7,7 @@ require ( github.com/elastic/go-docappender/v2 v2.3.0 github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/elastic/go-structform v0.0.12 + github.com/json-iterator/go v1.1.12 github.com/lestrrat-go/strftime v1.1.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.111.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.111.0 @@ -43,7 +44,6 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.10 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index 4bf95be05f3d..434bb1090a93 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -15,6 +15,7 @@ import ( "slices" "time" + jsoniter "github.com/json-iterator/go" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" @@ -65,6 +66,8 @@ var resourceAttrsToPreserve = map[string]bool{ semconv.AttributeHostName: true, } +var ErrInvalidTypeForBodyMapMode = errors.New("invalid log record body type for 'bodymap' mapping mode") + type mappingModel interface { encodeLog(pcommon.Resource, string, plog.LogRecord, pcommon.InstrumentationScope, string) ([]byte, error) encodeSpan(pcommon.Resource, string, ptrace.Span, pcommon.InstrumentationScope, string) ([]byte, error) @@ -107,6 +110,8 @@ func (m *encodeModel) encodeLog(resource pcommon.Resource, resourceSchemaURL str document = m.encodeLogECSMode(resource, record, scope) case MappingOTel: document = m.encodeLogOTelMode(resource, resourceSchemaURL, record, scope, scopeSchemaURL) + case MappingBodyMap: + return m.encodeLogBodyMapMode(record) default: document = m.encodeLogDefaultMode(resource, record, scope) } @@ -138,6 +143,15 @@ func (m *encodeModel) encodeLogDefaultMode(resource pcommon.Resource, record plo return document } +func (m *encodeModel) encodeLogBodyMapMode(record plog.LogRecord) ([]byte, error) { + body := record.Body() + if body.Type() != pcommon.ValueTypeMap { + return nil, fmt.Errorf("%w: %q", ErrInvalidTypeForBodyMapMode, body.Type()) + } + + return jsoniter.Marshal(body.Map().AsRaw()) +} + func (m *encodeModel) encodeLogOTelMode(resource pcommon.Resource, resourceSchemaURL string, record plog.LogRecord, scope pcommon.InstrumentationScope, scopeSchemaURL string) objmodel.Document { var document objmodel.Document diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go index 2bc34e9a24cf..136039cf28ea 100644 --- a/exporter/elasticsearchexporter/model_test.go +++ b/exporter/elasticsearchexporter/model_test.go @@ -1225,3 +1225,43 @@ func TestEncodeLogScalarObjectConflict(t *testing.T) { fooValue = gjson.GetBytes(encoded, "Attributes\\.foo\\.value") assert.Equal(t, "foovalue", fooValue.Str) } + +func TestEncodeLogBodyMapMode(t *testing.T) { + // craft a log record with a body map + logs := plog.NewLogs() + resourceLogs := logs.ResourceLogs().AppendEmpty() + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + logRecords := scopeLogs.LogRecords() + observedTimestamp := pcommon.Timestamp(time.Now().UnixNano()) + + logRecord := logRecords.AppendEmpty() + logRecord.SetObservedTimestamp(observedTimestamp) + + bodyMap := pcommon.NewMap() + bodyMap.PutStr("@timestamp", "2024-03-12T20:00:41.123456789Z") + bodyMap.PutInt("id", 1) + bodyMap.PutStr("key", "value") + bodyMap.PutStr("key.a", "a") + bodyMap.PutStr("key.a.b", "b") + bodyMap.PutDouble("pi", 3.14) + bodyMap.CopyTo(logRecord.Body().SetEmptyMap()) + + m := encodeModel{} + got, err := m.encodeLogBodyMapMode(logRecord) + require.NoError(t, err) + + require.JSONEq(t, `{ + "@timestamp": "2024-03-12T20:00:41.123456789Z", + "id": 1, + "key": "value", + "key.a": "a", + "key.a.b": "b", + "pi": 3.14 + }`, string(got)) + + // invalid body map + logRecord.Body().SetEmptySlice() + _, err = m.encodeLogBodyMapMode(logRecord) + require.Error(t, err) + require.ErrorIs(t, err, ErrInvalidTypeForBodyMapMode) +} From e9efb4088d7f935ba34830b96da29c1a351bfc86 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 17 Oct 2024 15:35:59 +0100 Subject: [PATCH 04/21] [exporter/elasticsearch] Preserve attribute names and metric names on prefix conflict in OTel mapping mode (#35651) #### Description Metric names should be flattened and exported as is, even when one metric name is a prefix of another. Same for attributes for all logs, metrics and traces. #### Link to tracking issue #### Testing #### Documentation --- ...ode-passthrough-field-prefix-conflict.yaml | 27 ++++ .../elasticsearchexporter/exporter_test.go | 127 ++++++++++++++++++ .../internal/objmodel/objmodel.go | 32 ++--- .../internal/objmodel/objmodel_test.go | 40 ++++-- exporter/elasticsearchexporter/model.go | 9 +- 5 files changed, 206 insertions(+), 29 deletions(-) create mode 100644 .chloggen/elasticsearchexporter_otel-mode-passthrough-field-prefix-conflict.yaml diff --git a/.chloggen/elasticsearchexporter_otel-mode-passthrough-field-prefix-conflict.yaml b/.chloggen/elasticsearchexporter_otel-mode-passthrough-field-prefix-conflict.yaml new file mode 100644 index 000000000000..afde47be348b --- /dev/null +++ b/.chloggen/elasticsearchexporter_otel-mode-passthrough-field-prefix-conflict.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Preserve attribute names and metric names on prefix conflict in OTel mapping mode + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35651] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: e.g. if there are attributes "a" and "a.b", they should be sent to Elasticsearch as is, instead of "a.value" and "a.b", in OTel mapping mode + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index e2871666b138..f1b455e41e1e 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -714,6 +714,35 @@ func TestExporterLogs(t *testing.T) { assert.Equal(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) assert.Equal(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) + + t.Run("otel mode attribute key prefix conflict", func(t *testing.T) { + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + return itemsAllOK(docs) + }) + + exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { + cfg.Mapping.Mode = "otel" + }) + + mustSendLogs(t, exporter, newLogsWithAttributes(map[string]any{ + "a": "a", + "a.b": "a.b", + }, map[string]any{ + "a": "a", + "a.b": "a.b", + }, map[string]any{ + "a": "a", + "a.b": "a.b", + })) + + rec.WaitItems(1) + doc := rec.Items()[0].Document + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `scope.attributes`).Raw) + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `resource.attributes`).Raw) + }) } func TestExporterMetrics(t *testing.T) { @@ -1300,6 +1329,75 @@ func TestExporterMetrics(t *testing.T) { assertItemsEqual(t, expected, rec.Items(), false) }) + t.Run("otel mode metric name conflict", func(t *testing.T) { + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + return itemsAllOK(docs) + }) + + exporter := newTestMetricsExporter(t, server.URL, func(cfg *Config) { + cfg.Mapping.Mode = "otel" + }) + + metrics := pmetric.NewMetrics() + resourceMetric := metrics.ResourceMetrics().AppendEmpty() + scopeMetric := resourceMetric.ScopeMetrics().AppendEmpty() + + fooBarMetric := scopeMetric.Metrics().AppendEmpty() + fooBarMetric.SetName("foo.bar") + fooBarMetric.SetEmptySum().DataPoints().AppendEmpty().SetIntValue(0) + + fooMetric := scopeMetric.Metrics().AppendEmpty() + fooMetric.SetName("foo") + fooMetric.SetEmptySum().DataPoints().AppendEmpty().SetIntValue(0) + + fooBarBazMetric := scopeMetric.Metrics().AppendEmpty() + fooBarBazMetric.SetName("foo.bar.baz") + fooBarBazMetric.SetEmptySum().DataPoints().AppendEmpty().SetIntValue(0) + + mustSendMetrics(t, exporter, metrics) + + rec.WaitItems(1) + expected := []itemRequest{ + { + Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.foo.bar":"gauge_long","metrics.foo":"gauge_long","metrics.foo.bar.baz":"gauge_long"}}}`), + Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"metrics":{"foo":0,"foo.bar":0,"foo.bar.baz":0},"resource":{"dropped_attributes_count":0},"scope":{"dropped_attributes_count":0}}`), + }, + } + + assertItemsEqual(t, expected, rec.Items(), false) + }) + + t.Run("otel mode attribute key prefix conflict", func(t *testing.T) { + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + return itemsAllOK(docs) + }) + + exporter := newTestMetricsExporter(t, server.URL, func(cfg *Config) { + cfg.Mapping.Mode = "otel" + }) + + mustSendMetrics(t, exporter, newMetricsWithAttributes(map[string]any{ + "a": "a", + "a.b": "a.b", + }, map[string]any{ + "a": "a", + "a.b": "a.b", + }, map[string]any{ + "a": "a", + "a.b": "a.b", + })) + + rec.WaitItems(1) + doc := rec.Items()[0].Document + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `scope.attributes`).Raw) + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `resource.attributes`).Raw) + }) + t.Run("publish summary", func(t *testing.T) { rec := newBulkRecorder() server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { @@ -1600,6 +1698,35 @@ func TestExporterTraces(t *testing.T) { assert.Equal(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) } }) + + t.Run("otel mode attribute key prefix conflict", func(t *testing.T) { + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + return itemsAllOK(docs) + }) + + exporter := newTestTracesExporter(t, server.URL, func(cfg *Config) { + cfg.Mapping.Mode = "otel" + }) + + mustSendTraces(t, exporter, newTracesWithAttributes(map[string]any{ + "a": "a", + "a.b": "a.b", + }, map[string]any{ + "a": "a", + "a.b": "a.b", + }, map[string]any{ + "a": "a", + "a.b": "a.b", + })) + + rec.WaitItems(1) + doc := rec.Items()[0].Document + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `scope.attributes`).Raw) + assert.JSONEq(t, `{"a":"a","a.b":"a.b"}`, gjson.GetBytes(doc, `resource.attributes`).Raw) + }) } // TestExporterAuth verifies that the Elasticsearch exporter supports diff --git a/exporter/elasticsearchexporter/internal/objmodel/objmodel.go b/exporter/elasticsearchexporter/internal/objmodel/objmodel.go index f20f9b1d213b..33a63abc794d 100644 --- a/exporter/elasticsearchexporter/internal/objmodel/objmodel.go +++ b/exporter/elasticsearchexporter/internal/objmodel/objmodel.go @@ -209,12 +209,12 @@ func (doc *Document) sort() { // The filtering only keeps the last value for a key. // // Dedup ensure that keys are sorted. -func (doc *Document) Dedup() { +func (doc *Document) Dedup(appendValueOnConflict bool) { // 1. Always ensure the fields are sorted, Dedup support requires // Fields to be sorted. doc.sort() - // 2. rename fields if a primitive value is overwritten by an object. + // 2. rename fields if a primitive value is overwritten by an object if appendValueOnConflict. // For example the pair (path.x=1, path.x.a="test") becomes: // (path.x.value=1, path.x.a="test"). // @@ -227,16 +227,18 @@ func (doc *Document) Dedup() { // field in favor of the `value` field in the document. // // This step removes potential conflicts when dedotting and serializing fields. - var renamed bool - for i := 0; i < len(doc.fields)-1; i++ { - key, nextKey := doc.fields[i].key, doc.fields[i+1].key - if len(key) < len(nextKey) && strings.HasPrefix(nextKey, key) && nextKey[len(key)] == '.' { - renamed = true - doc.fields[i].key = key + ".value" + if appendValueOnConflict { + var renamed bool + for i := 0; i < len(doc.fields)-1; i++ { + key, nextKey := doc.fields[i].key, doc.fields[i+1].key + if len(key) < len(nextKey) && strings.HasPrefix(nextKey, key) && nextKey[len(key)] == '.' { + renamed = true + doc.fields[i].key = key + ".value" + } + } + if renamed { + doc.sort() } - } - if renamed { - doc.sort() } // 3. mark duplicates as 'ignore' @@ -251,7 +253,7 @@ func (doc *Document) Dedup() { // 4. fix objects that might be stored in arrays for i := range doc.fields { - doc.fields[i].value.Dedup() + doc.fields[i].value.Dedup(appendValueOnConflict) } } @@ -487,13 +489,13 @@ func (v *Value) sort() { // Dedup recursively dedups keys in stored documents. // // NOTE: The value MUST be sorted. -func (v *Value) Dedup() { +func (v *Value) Dedup(appendValueOnConflict bool) { switch v.kind { case KindObject: - v.doc.Dedup() + v.doc.Dedup(appendValueOnConflict) case KindArr: for i := range v.arr { - v.arr[i].Dedup() + v.arr[i].Dedup(appendValueOnConflict) } } } diff --git a/exporter/elasticsearchexporter/internal/objmodel/objmodel_test.go b/exporter/elasticsearchexporter/internal/objmodel/objmodel_test.go index 1961f716db05..3d0a07b820d0 100644 --- a/exporter/elasticsearchexporter/internal/objmodel/objmodel_test.go +++ b/exporter/elasticsearchexporter/internal/objmodel/objmodel_test.go @@ -86,8 +86,9 @@ func TestObjectModel_CreateMap(t *testing.T) { func TestObjectModel_Dedup(t *testing.T) { tests := map[string]struct { - build func() Document - want Document + build func() Document + appendValueOnConflict bool + want Document }{ "no duplicates": { build: func() (doc Document) { @@ -95,7 +96,8 @@ func TestObjectModel_Dedup(t *testing.T) { doc.AddInt("c", 3) return doc }, - want: Document{fields: []field{{"a", IntValue(1)}, {"c", IntValue(3)}}}, + appendValueOnConflict: true, + want: Document{fields: []field{{"a", IntValue(1)}, {"c", IntValue(3)}}}, }, "duplicate keys": { build: func() (doc Document) { @@ -104,7 +106,8 @@ func TestObjectModel_Dedup(t *testing.T) { doc.AddInt("a", 2) return doc }, - want: Document{fields: []field{{"a", ignoreValue}, {"a", IntValue(2)}, {"c", IntValue(3)}}}, + appendValueOnConflict: true, + want: Document{fields: []field{{"a", ignoreValue}, {"a", IntValue(2)}, {"c", IntValue(3)}}}, }, "duplicate after flattening from map: namespace object at end": { build: func() Document { @@ -114,7 +117,8 @@ func TestObjectModel_Dedup(t *testing.T) { am.PutEmptyMap("namespace").PutInt("a", 23) return DocumentFromAttributes(am) }, - want: Document{fields: []field{{"namespace.a", ignoreValue}, {"namespace.a", IntValue(23)}, {"toplevel", StringValue("test")}}}, + appendValueOnConflict: true, + want: Document{fields: []field{{"namespace.a", ignoreValue}, {"namespace.a", IntValue(23)}, {"toplevel", StringValue("test")}}}, }, "duplicate after flattening from map: namespace object at beginning": { build: func() Document { @@ -124,7 +128,8 @@ func TestObjectModel_Dedup(t *testing.T) { am.PutStr("toplevel", "test") return DocumentFromAttributes(am) }, - want: Document{fields: []field{{"namespace.a", ignoreValue}, {"namespace.a", IntValue(42)}, {"toplevel", StringValue("test")}}}, + appendValueOnConflict: true, + want: Document{fields: []field{{"namespace.a", ignoreValue}, {"namespace.a", IntValue(42)}, {"toplevel", StringValue("test")}}}, }, "dedup in arrays": { build: func() (doc Document) { @@ -136,6 +141,7 @@ func TestObjectModel_Dedup(t *testing.T) { doc.Add("arr", ArrValue(Value{kind: KindObject, doc: embedded})) return doc }, + appendValueOnConflict: true, want: Document{fields: []field{{"arr", ArrValue(Value{kind: KindObject, doc: Document{fields: []field{ {"a", ignoreValue}, {"a", IntValue(2)}, @@ -148,7 +154,8 @@ func TestObjectModel_Dedup(t *testing.T) { doc.AddInt("namespace.a", 2) return doc }, - want: Document{fields: []field{{"namespace.a", IntValue(2)}, {"namespace.value", IntValue(1)}}}, + appendValueOnConflict: true, + want: Document{fields: []field{{"namespace.a", IntValue(2)}, {"namespace.value", IntValue(1)}}}, }, "dedup removes primitive if value exists": { build: func() (doc Document) { @@ -157,14 +164,25 @@ func TestObjectModel_Dedup(t *testing.T) { doc.AddInt("namespace.value", 3) return doc }, - want: Document{fields: []field{{"namespace.a", IntValue(2)}, {"namespace.value", ignoreValue}, {"namespace.value", IntValue(3)}}}, + appendValueOnConflict: true, + want: Document{fields: []field{{"namespace.a", IntValue(2)}, {"namespace.value", ignoreValue}, {"namespace.value", IntValue(3)}}}, + }, + "dedup without append value on conflict": { + build: func() (doc Document) { + doc.AddInt("namespace", 1) + doc.AddInt("namespace.a", 2) + doc.AddInt("namespace.value", 3) + return doc + }, + appendValueOnConflict: false, + want: Document{fields: []field{{"namespace", IntValue(1)}, {"namespace.a", IntValue(2)}, {"namespace.value", IntValue(3)}}}, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { doc := test.build() - doc.Dedup() + doc.Dedup(test.appendValueOnConflict) assert.Equal(t, test.want, doc) }) } @@ -282,7 +300,7 @@ func TestDocument_Serialize_Flat(t *testing.T) { m := pcommon.NewMap() assert.NoError(t, m.FromRaw(test.attrs)) doc := DocumentFromAttributes(m) - doc.Dedup() + doc.Dedup(true) err := doc.Serialize(&buf, false, false) require.NoError(t, err) @@ -343,7 +361,7 @@ func TestDocument_Serialize_Dedot(t *testing.T) { m := pcommon.NewMap() assert.NoError(t, m.FromRaw(test.attrs)) doc := DocumentFromAttributes(m) - doc.Dedup() + doc.Dedup(true) err := doc.Serialize(&buf, true, false) require.NoError(t, err) diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index 434bb1090a93..8c71df950752 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -115,7 +115,8 @@ func (m *encodeModel) encodeLog(resource pcommon.Resource, resourceSchemaURL str default: document = m.encodeLogDefaultMode(resource, record, scope) } - document.Dedup() + // For OTel mode, prefix conflicts are not a problem as otel-data has subobjects: false + document.Dedup(m.mode != MappingOTel) var buf bytes.Buffer err := document.Serialize(&buf, m.dedot, m.mode == MappingOTel) @@ -267,7 +268,8 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo } func (m *encodeModel) encodeDocument(document objmodel.Document) ([]byte, error) { - document.Dedup() + // For OTel mode, prefix conflicts are not a problem as otel-data has subobjects: false + document.Dedup(m.mode != MappingOTel) var buf bytes.Buffer err := document.Serialize(&buf, m.dedot, m.mode == MappingOTel) @@ -646,7 +648,8 @@ func (m *encodeModel) encodeSpan(resource pcommon.Resource, resourceSchemaURL st default: document = m.encodeSpanDefaultMode(resource, span, scope) } - document.Dedup() + // For OTel mode, prefix conflicts are not a problem as otel-data has subobjects: false + document.Dedup(m.mode != MappingOTel) var buf bytes.Buffer err := document.Serialize(&buf, m.dedot, m.mode == MappingOTel) return buf.Bytes(), err From d8cad1c50e8a086bdeb17991d618e2ad083d1e75 Mon Sep 17 00:00:00 2001 From: Florian Bacher Date: Thu, 17 Oct 2024 18:02:29 +0200 Subject: [PATCH 05/21] [extension/opamp] Implement ReportsHealth capability (#35488) --- .chloggen/opamp-extension-reportshealth.yaml | 27 +++++++++++++ extension/opampextension/README.md | 1 + extension/opampextension/config.go | 5 +++ extension/opampextension/config_test.go | 41 ++++++++++++++++++++ extension/opampextension/factory.go | 1 + extension/opampextension/opamp_agent.go | 23 +++++++++++ extension/opampextension/opamp_agent_test.go | 1 + 7 files changed, 99 insertions(+) create mode 100644 .chloggen/opamp-extension-reportshealth.yaml diff --git a/.chloggen/opamp-extension-reportshealth.yaml b/.chloggen/opamp-extension-reportshealth.yaml new file mode 100644 index 000000000000..c58f185a6628 --- /dev/null +++ b/.chloggen/opamp-extension-reportshealth.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: opampextension + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Implement `ReportsHealth` capability in OpAMP extension + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35433] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/extension/opampextension/README.md b/extension/opampextension/README.md index 59d7284fc643..4d5227c2e276 100644 --- a/extension/opampextension/README.md +++ b/extension/opampextension/README.md @@ -43,6 +43,7 @@ The following settings are optional for both transports: instance UID remains constant across process restarts. - `capabilities`: Keys with boolean true/false values that enable a particular OpAMP capability. - `reports_effective_config`: Whether to enable the OpAMP ReportsEffectiveConfig capability. Default is `true`. + - `reports_health`: Whether to enable the OpAMP ReportsHealth capability. Default is `true`. - `agent_description`: Setting that modifies the agent description reported to the OpAMP server. - `non_identifying_attributes`: A map of key value pairs that will be added to the [non-identifying attributes](https://github.com/open-telemetry/opamp-spec/blob/main/specification.md#agentdescriptionnon_identifying_attributes) reported to the OpAMP server. If an attribute collides with the default non-identifying attributes that are automatically added, the ones specified here take precedence. - `ppid`: An optional process ID to monitor. When this process is no longer running, the extension will emit a fatal error, causing the collector to exit. This is meant to be set by the Supervisor or some other parent process, and should not be configured manually. diff --git a/extension/opampextension/config.go b/extension/opampextension/config.go index a06adee4fab8..e47ae1894ed0 100644 --- a/extension/opampextension/config.go +++ b/extension/opampextension/config.go @@ -54,6 +54,8 @@ type AgentDescription struct { type Capabilities struct { // ReportsEffectiveConfig enables the OpAMP ReportsEffectiveConfig Capability. (default: true) ReportsEffectiveConfig bool `mapstructure:"reports_effective_config"` + // ReportsHealth enables the OpAMP ReportsHealth Capability. (default: true) + ReportsHealth bool `mapstructure:"reports_health"` } func (caps Capabilities) toAgentCapabilities() protobufs.AgentCapabilities { @@ -63,6 +65,9 @@ func (caps Capabilities) toAgentCapabilities() protobufs.AgentCapabilities { if caps.ReportsEffectiveConfig { agentCapabilities |= protobufs.AgentCapabilities_AgentCapabilities_ReportsEffectiveConfig } + if caps.ReportsHealth { + agentCapabilities |= protobufs.AgentCapabilities_AgentCapabilities_ReportsHealth + } return agentCapabilities } diff --git a/extension/opampextension/config_test.go b/extension/opampextension/config_test.go index bbccff4d91aa..7f50970f3184 100644 --- a/extension/opampextension/config_test.go +++ b/extension/opampextension/config_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/open-telemetry/opamp-go/protobufs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/config/configopaque" @@ -39,6 +40,7 @@ func TestUnmarshalConfig(t *testing.T) { InstanceUID: "01BX5ZZKBKACTAV9WEVGEMMVRZ", Capabilities: Capabilities{ ReportsEffectiveConfig: true, + ReportsHealth: true, }, PPIDPollInterval: 5 * time.Second, }, cfg) @@ -63,6 +65,7 @@ func TestUnmarshalHttpConfig(t *testing.T) { InstanceUID: "01BX5ZZKBKACTAV9WEVGEMMVRZ", Capabilities: Capabilities{ ReportsEffectiveConfig: true, + ReportsHealth: true, }, PPIDPollInterval: 5 * time.Second, }, cfg) @@ -286,3 +289,41 @@ func TestConfig_Validate(t *testing.T) { }) } } + +func TestCapabilities_toAgentCapabilities(t *testing.T) { + type fields struct { + ReportsEffectiveConfig bool + ReportsHealth bool + } + tests := []struct { + name string + fields fields + want protobufs.AgentCapabilities + }{ + { + name: "default capabilities", + fields: fields{ + ReportsEffectiveConfig: false, + ReportsHealth: false, + }, + want: protobufs.AgentCapabilities_AgentCapabilities_ReportsStatus, + }, + { + name: "all supported capabilities enabled", + fields: fields{ + ReportsEffectiveConfig: true, + ReportsHealth: true, + }, + want: protobufs.AgentCapabilities_AgentCapabilities_ReportsStatus | protobufs.AgentCapabilities_AgentCapabilities_ReportsEffectiveConfig | protobufs.AgentCapabilities_AgentCapabilities_ReportsHealth, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + caps := Capabilities{ + ReportsEffectiveConfig: tt.fields.ReportsEffectiveConfig, + ReportsHealth: tt.fields.ReportsHealth, + } + assert.Equalf(t, tt.want, caps.toAgentCapabilities(), "toAgentCapabilities()") + }) + } +} diff --git a/extension/opampextension/factory.go b/extension/opampextension/factory.go index ea4ea23a22d6..868c3bc85c65 100644 --- a/extension/opampextension/factory.go +++ b/extension/opampextension/factory.go @@ -27,6 +27,7 @@ func createDefaultConfig() component.Config { Server: &OpAMPServer{}, Capabilities: Capabilities{ ReportsEffectiveConfig: true, + ReportsHealth: true, }, PPIDPollInterval: 5 * time.Second, } diff --git a/extension/opampextension/opamp_agent.go b/extension/opampextension/opamp_agent.go index db1ef789e738..f984974c6c88 100644 --- a/extension/opampextension/opamp_agent.go +++ b/extension/opampextension/opamp_agent.go @@ -32,6 +32,8 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/extension/opampcustommessages" ) +var _ extensioncapabilities.PipelineWatcher = (*opampAgent)(nil) + type opampAgent struct { cfg *Config logger *zap.Logger @@ -121,6 +123,8 @@ func (o *opampAgent) Start(ctx context.Context, host component.Host) error { return err } + o.setHealth(&protobufs.ComponentHealth{Healthy: false}) + o.logger.Debug("Starting OpAMP client...") if err := o.opampClient.Start(context.Background(), settings); err != nil { @@ -141,6 +145,7 @@ func (o *opampAgent) Shutdown(ctx context.Context) error { if o.opampClient == nil { return nil } + o.logger.Debug("Stopping OpAMP client...") err := o.opampClient.Stop(ctx) // Opamp-go considers this an error, but the collector does not. @@ -178,6 +183,16 @@ func (o *opampAgent) Register(capability string, opts ...opampcustommessages.Cus return o.customCapabilityRegistry.Register(capability, opts...) } +func (o *opampAgent) Ready() error { + o.setHealth(&protobufs.ComponentHealth{Healthy: true}) + return nil +} + +func (o *opampAgent) NotReady() error { + o.setHealth(&protobufs.ComponentHealth{Healthy: false}) + return nil +} + func (o *opampAgent) updateEffectiveConfig(conf *confmap.Conf) { o.eclk.Lock() defer o.eclk.Unlock() @@ -344,3 +359,11 @@ func (o *opampAgent) onMessage(_ context.Context, msg *types.MessageData) { o.customCapabilityRegistry.ProcessMessage(msg.CustomMessage) } } + +func (o *opampAgent) setHealth(ch *protobufs.ComponentHealth) { + if o.capabilities.ReportsHealth && o.opampClient != nil { + if err := o.opampClient.SetHealth(ch); err != nil { + o.logger.Error("Could not report health to OpAMP server", zap.Error(err)) + } + } +} diff --git a/extension/opampextension/opamp_agent_test.go b/extension/opampextension/opamp_agent_test.go index e2013d1d45eb..fd72d346492c 100644 --- a/extension/opampextension/opamp_agent_test.go +++ b/extension/opampextension/opamp_agent_test.go @@ -31,6 +31,7 @@ func TestNewOpampAgent(t *testing.T) { assert.Equal(t, "test version", o.agentVersion) assert.NotEmpty(t, o.instanceID.String()) assert.True(t, o.capabilities.ReportsEffectiveConfig) + assert.True(t, o.capabilities.ReportsHealth) assert.Empty(t, o.effectiveConfig) assert.Nil(t, o.agentDescription) } From 7d02d77ecab748221f3d2339581678c27841180e Mon Sep 17 00:00:00 2001 From: Stefan Kurek Date: Thu, 17 Oct 2024 12:28:24 -0400 Subject: [PATCH 06/21] Adds replication metrics to versions of MySQL older than 8.0.22 (#35776) #### Description Version of MySQL older than 8.0.22 use the `show slave status` vs `show replica status` view to retrieve replication metrics. This allows for both the newer & older views to work and retrieve data for the two actual replica metrics available (`mysql.replica.time_behind_source` & `mysql.replica.sql_delay`). #### Link to tracking issue Fixes #35217 #### Testing Manual testing against MySQL versions 8.0, 8.0.37, & MariaDB Ver 15.1 Distrib 10.5.26-MariaDB #### Documentation None needed --- ...eceiver_older_replica_metrics_support.yaml | 27 ++ receiver/mysqlreceiver/client.go | 255 +++++++++++++----- receiver/mysqlreceiver/go.mod | 1 + receiver/mysqlreceiver/go.sum | 2 + receiver/mysqlreceiver/scraper_test.go | 6 +- 5 files changed, 218 insertions(+), 73 deletions(-) create mode 100644 .chloggen/mysqlreceiver_older_replica_metrics_support.yaml diff --git a/.chloggen/mysqlreceiver_older_replica_metrics_support.yaml b/.chloggen/mysqlreceiver_older_replica_metrics_support.yaml new file mode 100644 index 000000000000..aecdf987b857 --- /dev/null +++ b/.chloggen/mysqlreceiver_older_replica_metrics_support.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: mysqlreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add replica metric support for versions of MySQL earlier than 8.0.22. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35217] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/mysqlreceiver/client.go b/receiver/mysqlreceiver/client.go index 736257e6a623..3aeabf1b777b 100644 --- a/receiver/mysqlreceiver/client.go +++ b/receiver/mysqlreceiver/client.go @@ -12,11 +12,12 @@ import ( // registers the mysql driver "github.com/go-sql-driver/mysql" + "github.com/hashicorp/go-version" ) type client interface { Connect() error - getVersion() (string, error) + getVersion() (*version.Version, error) getGlobalStats() (map[string]string, error) getInnodbStats() (map[string]string, error) getTableStats() ([]TableStats, error) @@ -110,66 +111,83 @@ type tableLockWaitEventStats struct { } type ReplicaStatusStats struct { - replicaIOState string - sourceHost string - sourceUser string - sourcePort int64 - connectRetry int64 - sourceLogFile string - readSourceLogPos int64 - relayLogFile string - relayLogPos int64 - relaySourceLogFile string - replicaIORunning string - replicaSQLRunning string - replicateDoDB string - replicateIgnoreDB string - replicateDoTable string - replicateIgnoreTable string - replicateWildDoTable string - replicateWildIgnoreTable string - lastErrno int64 - lastError string - skipCounter int64 - execSourceLogPos int64 - relayLogSpace int64 - untilCondition string - untilLogFile string - untilLogPos string - sourceSSLAllowed string - sourceSSLCAFile string - sourceSSLCAPath string - sourceSSLCert string - sourceSSLCipher string - sourceSSLKey string - secondsBehindSource sql.NullInt64 - sourceSSLVerifyServerCert string - lastIOErrno int64 - lastIOError string - lastSQLErrno int64 - lastSQLError string - replicateIgnoreServerIDs string - sourceServerID int64 - sourceUUID string - sourceInfoFile string - sqlDelay int64 - sqlRemainingDelay sql.NullInt64 - replicaSQLRunningState string - sourceRetryCount int64 - sourceBind string - lastIOErrorTimestamp string - lastSQLErrorTimestamp string - sourceSSLCrl string - sourceSSLCrlpath string - retrievedGtidSet string - executedGtidSet string - autoPosition string - replicateRewriteDB string - channelName string - sourceTLSVersion string - sourcePublicKeyPath string - getSourcePublicKey int64 - networkNamespace string + replicaIOState string + sourceHost string + sourceUser string + sourcePort int64 + connectRetry int64 + sourceLogFile string + readSourceLogPos int64 + relayLogFile string + relayLogPos int64 + relaySourceLogFile string + replicaIORunning string + replicaSQLRunning string + replicateDoDB string + replicateIgnoreDB string + replicateDoTable string + replicateIgnoreTable string + replicateWildDoTable string + replicateWildIgnoreTable string + lastErrno int64 + lastError string + skipCounter int64 + execSourceLogPos int64 + relayLogSpace int64 + untilCondition string + untilLogFile string + untilLogPos string + sourceSSLAllowed string + sourceSSLCAFile string + sourceSSLCAPath string + sourceSSLCert string + sourceSSLCipher string + sourceSSLKey string + secondsBehindSource sql.NullInt64 + sourceSSLVerifyServerCert string + lastIOErrno int64 + lastIOError string + lastSQLErrno int64 + lastSQLError string + replicateIgnoreServerIDs string + sourceServerID int64 + sourceUUID string + sourceInfoFile string + sqlDelay int64 + sqlRemainingDelay sql.NullInt64 + replicaSQLRunningState string + sourceRetryCount int64 + sourceBind string + lastIOErrorTimestamp string + lastSQLErrorTimestamp string + sourceSSLCrl string + sourceSSLCrlpath string + retrievedGtidSet string + executedGtidSet string + autoPosition string + replicateRewriteDB string + channelName string + sourceTLSVersion string + sourcePublicKeyPath string + getSourcePublicKey int64 + networkNamespace string + usingGtid string + gtidIoPos string + slaveDdlGroups int64 + slaveNonTransactionalGroups int64 + slaveTransactionalGroups int64 + retriedTransactions int64 + maxRelayLogSize int64 + executedLogEntries int64 + slaveReceivedHeartbeats int64 + slaveHeartbeatPeriod int64 + gtidSlavePos string + masterLastEventTime string + slaveLastEventTime string + masterSlaveTimeDiff string + parallelMode string + replicateDoDomainIDs string + replicateIgnoreDomainIDs string } var _ client = (*mySQLClient)(nil) @@ -218,15 +236,15 @@ func (c *mySQLClient) Connect() error { } // getVersion queries the db for the version. -func (c *mySQLClient) getVersion() (string, error) { +func (c *mySQLClient) getVersion() (*version.Version, error) { query := "SELECT VERSION();" - var version string - err := c.client.QueryRow(query).Scan(&version) + var versionStr string + err := c.client.QueryRow(query).Scan(&versionStr) if err != nil { - return "", err + return nil, err } - - return version, nil + version, err := version.NewVersion(versionStr) + return version, err } // getGlobalStats queries the db for global status metrics. @@ -397,16 +415,19 @@ func (c *mySQLClient) getTableLockWaitEventStats() ([]tableLockWaitEventStats, e } func (c *mySQLClient) getReplicaStatusStats() ([]ReplicaStatusStats, error) { - version, err := c.getVersion() + mysqlVersion, err := c.getVersion() if err != nil { return nil, err } - if version < "8.0.22" { - return nil, nil + query := "SHOW REPLICA STATUS" + minMysqlVersion, _ := version.NewVersion("8.0.22") + if strings.Contains(mysqlVersion.String(), "MariaDB") { + query = "SHOW SLAVE STATUS" + } else if mysqlVersion.LessThan(minMysqlVersion) { + query = "SHOW SLAVE STATUS" } - query := "SHOW REPLICA STATUS" rows, err := c.client.Query(query) if err != nil { @@ -427,28 +448,46 @@ func (c *mySQLClient) getReplicaStatusStats() ([]ReplicaStatusStats, error) { switch strings.ToLower(col) { case "replica_io_state": dest = append(dest, &s.replicaIOState) + case "slave_io_state": + dest = append(dest, &s.replicaIOState) case "source_host": dest = append(dest, &s.sourceHost) + case "master_host": + dest = append(dest, &s.sourceHost) case "source_user": dest = append(dest, &s.sourceUser) + case "master_user": + dest = append(dest, &s.sourceUser) case "source_port": dest = append(dest, &s.sourcePort) + case "master_port": + dest = append(dest, &s.sourcePort) case "connect_retry": dest = append(dest, &s.connectRetry) case "source_log_file": dest = append(dest, &s.sourceLogFile) + case "master_log_file": + dest = append(dest, &s.sourceLogFile) case "read_source_log_pos": dest = append(dest, &s.readSourceLogPos) + case "read_master_log_pos": + dest = append(dest, &s.readSourceLogPos) case "relay_log_file": dest = append(dest, &s.relayLogFile) case "relay_log_pos": dest = append(dest, &s.relayLogPos) case "relay_source_log_file": dest = append(dest, &s.relaySourceLogFile) + case "relay_master_log_file": + dest = append(dest, &s.relaySourceLogFile) case "replica_io_running": dest = append(dest, &s.replicaIORunning) + case "slave_io_running": + dest = append(dest, &s.replicaIORunning) case "replica_sql_running": dest = append(dest, &s.replicaSQLRunning) + case "slave_sql_running": + dest = append(dest, &s.replicaSQLRunning) case "replicate_do_db": dest = append(dest, &s.replicateDoDB) case "replicate_ignore_db": @@ -469,6 +508,8 @@ func (c *mySQLClient) getReplicaStatusStats() ([]ReplicaStatusStats, error) { dest = append(dest, &s.skipCounter) case "exec_source_log_pos": dest = append(dest, &s.execSourceLogPos) + case "exec_master_log_pos": + dest = append(dest, &s.execSourceLogPos) case "relay_log_space": dest = append(dest, &s.relayLogSpace) case "until_condition": @@ -479,20 +520,36 @@ func (c *mySQLClient) getReplicaStatusStats() ([]ReplicaStatusStats, error) { dest = append(dest, &s.untilLogPos) case "source_ssl_allowed": dest = append(dest, &s.sourceSSLAllowed) + case "master_ssl_allowed": + dest = append(dest, &s.sourceSSLAllowed) case "source_ssl_ca_file": dest = append(dest, &s.sourceSSLCAFile) + case "master_ssl_ca_file": + dest = append(dest, &s.sourceSSLCAFile) case "source_ssl_ca_path": dest = append(dest, &s.sourceSSLCAPath) + case "master_ssl_ca_path": + dest = append(dest, &s.sourceSSLCAPath) case "source_ssl_cert": dest = append(dest, &s.sourceSSLCert) + case "master_ssl_cert": + dest = append(dest, &s.sourceSSLCert) case "source_ssl_cipher": dest = append(dest, &s.sourceSSLCipher) + case "master_ssl_cipher": + dest = append(dest, &s.sourceSSLCipher) case "source_ssl_key": dest = append(dest, &s.sourceSSLKey) + case "master_ssl_key": + dest = append(dest, &s.sourceSSLKey) case "seconds_behind_source": dest = append(dest, &s.secondsBehindSource) + case "seconds_behind_master": + dest = append(dest, &s.secondsBehindSource) case "source_ssl_verify_server_cert": dest = append(dest, &s.sourceSSLVerifyServerCert) + case "master_ssl_verify_server_cert": + dest = append(dest, &s.sourceSSLVerifyServerCert) case "last_io_errno": dest = append(dest, &s.lastIOErrno) case "last_io_error": @@ -505,28 +562,44 @@ func (c *mySQLClient) getReplicaStatusStats() ([]ReplicaStatusStats, error) { dest = append(dest, &s.replicateIgnoreServerIDs) case "source_server_id": dest = append(dest, &s.sourceServerID) + case "master_server_id": + dest = append(dest, &s.sourceServerID) case "source_uuid": dest = append(dest, &s.sourceUUID) + case "master_uuid": + dest = append(dest, &s.sourceUUID) case "source_info_file": dest = append(dest, &s.sourceInfoFile) + case "master_info_file": + dest = append(dest, &s.sourceInfoFile) case "sql_delay": dest = append(dest, &s.sqlDelay) case "sql_remaining_delay": dest = append(dest, &s.sqlRemainingDelay) case "replica_sql_running_state": dest = append(dest, &s.replicaSQLRunningState) + case "slave_sql_running_state": + dest = append(dest, &s.replicaSQLRunningState) case "source_retry_count": dest = append(dest, &s.sourceRetryCount) + case "master_retry_count": + dest = append(dest, &s.sourceRetryCount) case "source_bind": dest = append(dest, &s.sourceBind) + case "master_bind": + dest = append(dest, &s.sourceBind) case "last_io_error_timestamp": dest = append(dest, &s.lastIOErrorTimestamp) case "last_sql_error_timestamp": dest = append(dest, &s.lastSQLErrorTimestamp) case "source_ssl_crl": dest = append(dest, &s.sourceSSLCrl) + case "master_ssl_crl": + dest = append(dest, &s.sourceSSLCrl) case "source_ssl_crlpath": dest = append(dest, &s.sourceSSLCrlpath) + case "master_ssl_crlpath": + dest = append(dest, &s.sourceSSLCrlpath) case "retrieved_gtid_set": dest = append(dest, &s.retrievedGtidSet) case "executed_gtid_set": @@ -539,12 +612,52 @@ func (c *mySQLClient) getReplicaStatusStats() ([]ReplicaStatusStats, error) { dest = append(dest, &s.channelName) case "source_tls_version": dest = append(dest, &s.sourceTLSVersion) + case "master_tls_version": + dest = append(dest, &s.sourceTLSVersion) case "source_public_key_path": dest = append(dest, &s.sourcePublicKeyPath) + case "master_public_key_path": + dest = append(dest, &s.sourcePublicKeyPath) case "get_source_public_key": dest = append(dest, &s.getSourcePublicKey) + case "get_master_public_key": + dest = append(dest, &s.getSourcePublicKey) case "network_namespace": dest = append(dest, &s.networkNamespace) + case "using_gtid": + dest = append(dest, &s.usingGtid) + case "gtid_io_pos": + dest = append(dest, &s.gtidIoPos) + case "slave_ddl_groups": + dest = append(dest, &s.slaveDdlGroups) + case "slave_non_transactional_groups": + dest = append(dest, &s.slaveNonTransactionalGroups) + case "slave_transactional_groups": + dest = append(dest, &s.slaveTransactionalGroups) + case "retried_transactions": + dest = append(dest, &s.retriedTransactions) + case "max_relay_log_size": + dest = append(dest, &s.maxRelayLogSize) + case "executed_log_entries": + dest = append(dest, &s.executedLogEntries) + case "slave_received_heartbeats": + dest = append(dest, &s.slaveReceivedHeartbeats) + case "slave_heartbeat_period": + dest = append(dest, &s.slaveHeartbeatPeriod) + case "gtid_slave_pos": + dest = append(dest, &s.gtidSlavePos) + case "master_last_event_time": + dest = append(dest, &s.masterLastEventTime) + case "slave_last_event_time": + dest = append(dest, &s.slaveLastEventTime) + case "master_slave_time_diff": + dest = append(dest, &s.masterSlaveTimeDiff) + case "parallel_mode": + dest = append(dest, &s.parallelMode) + case "replicate_do_domain_ids": + dest = append(dest, &s.replicateDoDomainIDs) + case "replicate_ignore_domain_ids": + dest = append(dest, &s.replicateIgnoreDomainIDs) default: return nil, fmt.Errorf("unknown column name %s for replica status", col) } diff --git a/receiver/mysqlreceiver/go.mod b/receiver/mysqlreceiver/go.mod index 44c73fe67420..9f70a56d04ab 100644 --- a/receiver/mysqlreceiver/go.mod +++ b/receiver/mysqlreceiver/go.mod @@ -48,6 +48,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect diff --git a/receiver/mysqlreceiver/go.sum b/receiver/mysqlreceiver/go.sum index cf30c4b18167..a054b1a95e2a 100644 --- a/receiver/mysqlreceiver/go.sum +++ b/receiver/mysqlreceiver/go.sum @@ -58,6 +58,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/receiver/mysqlreceiver/scraper_test.go b/receiver/mysqlreceiver/scraper_test.go index 01dc4dd11840..8753752e948a 100644 --- a/receiver/mysqlreceiver/scraper_test.go +++ b/receiver/mysqlreceiver/scraper_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/config/confignet" @@ -158,8 +159,9 @@ func (c *mockClient) Connect() error { return nil } -func (c *mockClient) getVersion() (string, error) { - return "8.0.27", nil +func (c *mockClient) getVersion() (*version.Version, error) { + version, _ := version.NewVersion("8.0.27") + return version, nil } func (c *mockClient) getGlobalStats() (map[string]string, error) { From 5b1b4d42c63ae90199d5d14144bf846b18f94726 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 17 Oct 2024 19:01:10 +0100 Subject: [PATCH 07/21] [exporter/elasticsearch] Deprecate retry::max_requests in favor of retry::max_retries (#35571) **Description:** The new retry::max_retries will be exactly retry::max_requests - 1, but it will be much more intuitive to the end user. Deprecate retry::max_requests. **Link to tracking Issue:** Fixes #32344 **Testing:** **Documentation:** --- ...hexporter_deprecate-retry-maxrequests.yaml | 27 ++++++++++++ exporter/elasticsearchexporter/README.md | 3 +- exporter/elasticsearchexporter/bulkindexer.go | 43 ++++++++----------- exporter/elasticsearchexporter/config.go | 24 ++++++++++- exporter/elasticsearchexporter/config_test.go | 14 ++++-- exporter/elasticsearchexporter/esclient.go | 17 +++----- .../elasticsearchexporter/exporter_test.go | 10 ++--- exporter/elasticsearchexporter/factory.go | 8 ++-- .../testdata/config.yaml | 6 +-- 9 files changed, 97 insertions(+), 55 deletions(-) create mode 100644 .chloggen/elasticsearchexporter_deprecate-retry-maxrequests.yaml diff --git a/.chloggen/elasticsearchexporter_deprecate-retry-maxrequests.yaml b/.chloggen/elasticsearchexporter_deprecate-retry-maxrequests.yaml new file mode 100644 index 000000000000..80bb0eac5fb8 --- /dev/null +++ b/.chloggen/elasticsearchexporter_deprecate-retry-maxrequests.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: deprecation + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Deprecate retry::max_requests in favor of retry::max_retries + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [32344] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: retry::max_retries will be exactly retry::max_requests - 1 + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 5ec203f13674..b620b81158e9 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -202,7 +202,8 @@ The behaviour of this bulk indexing can be configured with the following setting - `interval` (default=30s): Write buffer flush time limit. - `retry`: Elasticsearch bulk request retry settings - `enabled` (default=true): Enable/Disable request retry on error. Failed requests are retried with exponential backoff. - - `max_requests` (default=3): Number of HTTP request retries. + - `max_requests` (DEPRECATED, use retry::max_retries instead): Number of HTTP request retries including the initial attempt. If used, `retry::max_retries` will be set to `max_requests - 1`. + - `max_retries` (default=2): Number of HTTP request retries. To disable retries, set `retry::enabled` to `false` instead of setting `max_retries` to `0`. - `initial_interval` (default=100ms): Initial waiting time if a HTTP request failed. - `max_interval` (default=1m): Max waiting time if a HTTP request failed. - `retry_on_status` (default=[429]): Status codes that trigger request or document level retries. Request level retry and document level retry status codes are shared and cannot be configured separately. To avoid duplicates, it defaults to `[429]`. diff --git a/exporter/elasticsearchexporter/bulkindexer.go b/exporter/elasticsearchexporter/bulkindexer.go index 471ddc2dc7b9..62bc329a26f8 100644 --- a/exporter/elasticsearchexporter/bulkindexer.go +++ b/exporter/elasticsearchexporter/bulkindexer.go @@ -51,6 +51,8 @@ type bulkIndexerSession interface { Flush(context.Context) error } +const defaultMaxRetries = 2 + func newBulkIndexer(logger *zap.Logger, client *elasticsearch.Client, config *Config) (bulkIndexer, error) { if config.Batcher.Enabled != nil { return newSyncBulkIndexer(logger, client, config), nil @@ -58,20 +60,25 @@ func newBulkIndexer(logger *zap.Logger, client *elasticsearch.Client, config *Co return newAsyncBulkIndexer(logger, client, config) } -func newSyncBulkIndexer(logger *zap.Logger, client *elasticsearch.Client, config *Config) *syncBulkIndexer { - var maxDocRetry int +func bulkIndexerConfig(client *elasticsearch.Client, config *Config) docappender.BulkIndexerConfig { + var maxDocRetries int if config.Retry.Enabled { - // max_requests includes initial attempt - // See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/32344 - maxDocRetry = config.Retry.MaxRequests - 1 + maxDocRetries = defaultMaxRetries + if config.Retry.MaxRetries != 0 { + maxDocRetries = config.Retry.MaxRetries + } } + return docappender.BulkIndexerConfig{ + Client: client, + MaxDocumentRetries: maxDocRetries, + Pipeline: config.Pipeline, + RetryOnDocumentStatus: config.Retry.RetryOnStatus, + } +} + +func newSyncBulkIndexer(logger *zap.Logger, client *elasticsearch.Client, config *Config) *syncBulkIndexer { return &syncBulkIndexer{ - config: docappender.BulkIndexerConfig{ - Client: client, - MaxDocumentRetries: maxDocRetry, - Pipeline: config.Pipeline, - RetryOnDocumentStatus: config.Retry.RetryOnStatus, - }, + config: bulkIndexerConfig(client, config), flushTimeout: config.Timeout, retryConfig: config.Retry, logger: logger, @@ -165,13 +172,6 @@ func newAsyncBulkIndexer(logger *zap.Logger, client *elasticsearch.Client, confi flushBytes = 5e+6 } - var maxDocRetry int - if config.Retry.Enabled { - // max_requests includes initial attempt - // See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/32344 - maxDocRetry = config.Retry.MaxRequests - 1 - } - pool := &asyncBulkIndexer{ wg: sync.WaitGroup{}, items: make(chan docappender.BulkIndexerItem, config.NumWorkers), @@ -180,12 +180,7 @@ func newAsyncBulkIndexer(logger *zap.Logger, client *elasticsearch.Client, confi pool.wg.Add(numWorkers) for i := 0; i < numWorkers; i++ { - bi, err := docappender.NewBulkIndexer(docappender.BulkIndexerConfig{ - Client: client, - MaxDocumentRetries: maxDocRetry, - Pipeline: config.Pipeline, - RetryOnDocumentStatus: config.Retry.RetryOnStatus, - }) + bi, err := docappender.NewBulkIndexer(bulkIndexerConfig(client, config)) if err != nil { return nil, err } diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 072bd725c6fe..fe794d6db430 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -169,9 +169,13 @@ type RetrySettings struct { // Enabled allows users to disable retry without having to comment out all settings. Enabled bool `mapstructure:"enabled"` - // MaxRequests configures how often an HTTP request is retried before it is assumed to be failed. + // MaxRequests configures how often an HTTP request is attempted before it is assumed to be failed. + // Deprecated: use MaxRetries instead. MaxRequests int `mapstructure:"max_requests"` + // MaxRetries configures how many times an HTTP request is retried. + MaxRetries int `mapstructure:"max_retries"` + // InitialInterval configures the initial waiting time if a request failed. InitialInterval time.Duration `mapstructure:"initial_interval"` @@ -273,6 +277,17 @@ func (cfg *Config) Validate() error { // TODO support confighttp.ClientConfig.Compression return errors.New("compression is not currently configurable") } + + if cfg.Retry.MaxRequests != 0 && cfg.Retry.MaxRetries != 0 { + return errors.New("must not specify both retry::max_requests and retry::max_retries") + } + if cfg.Retry.MaxRequests < 0 { + return errors.New("retry::max_requests should be non-negative") + } + if cfg.Retry.MaxRetries < 0 { + return errors.New("retry::max_retries should be non-negative") + } + return nil } @@ -355,11 +370,16 @@ func (cfg *Config) MappingMode() MappingMode { return mappingModes[cfg.Mapping.Mode] } -func logConfigDeprecationWarnings(cfg *Config, logger *zap.Logger) { +func handleDeprecatedConfig(cfg *Config, logger *zap.Logger) { if cfg.Mapping.Dedup != nil { logger.Warn("dedup is deprecated, and is always enabled") } if cfg.Mapping.Dedot && cfg.MappingMode() != MappingECS || !cfg.Mapping.Dedot && cfg.MappingMode() == MappingECS { logger.Warn("dedot has been deprecated: in the future, dedotting will always be performed in ECS mode only") } + if cfg.Retry.MaxRequests != 0 { + cfg.Retry.MaxRetries = cfg.Retry.MaxRequests - 1 + // Do not set cfg.Retry.Enabled = false if cfg.Retry.MaxRequest = 1 to avoid breaking change on behavior + logger.Warn("retry::max_requests has been deprecated, and will be removed in a future version. Use retry::max_retries instead.") + } } diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index a27a28ccfe6e..9934dbb7365b 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -94,7 +94,7 @@ func TestConfig(t *testing.T) { }, Retry: RetrySettings{ Enabled: true, - MaxRequests: 5, + MaxRetries: 5, InitialInterval: 100 * time.Millisecond, MaxInterval: 1 * time.Minute, RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, @@ -164,7 +164,7 @@ func TestConfig(t *testing.T) { }, Retry: RetrySettings{ Enabled: true, - MaxRequests: 5, + MaxRetries: 5, InitialInterval: 100 * time.Millisecond, MaxInterval: 1 * time.Minute, RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, @@ -234,7 +234,7 @@ func TestConfig(t *testing.T) { }, Retry: RetrySettings{ Enabled: true, - MaxRequests: 5, + MaxRetries: 5, InitialInterval: 100 * time.Millisecond, MaxInterval: 1 * time.Minute, RetryOnStatus: []int{http.StatusTooManyRequests, http.StatusInternalServerError}, @@ -391,6 +391,14 @@ func TestConfig_Validate(t *testing.T) { }), err: `compression is not currently configurable`, }, + "both max_retries and max_requests specified": { + config: withDefaultConfig(func(cfg *Config) { + cfg.Endpoints = []string{"http://test:9200"} + cfg.Retry.MaxRetries = 1 + cfg.Retry.MaxRequests = 1 + }), + err: `must not specify both retry::max_requests and retry::max_retries`, + }, } for name, tt := range tests { diff --git a/exporter/elasticsearchexporter/esclient.go b/exporter/elasticsearchexporter/esclient.go index 23c2d48bb9ef..556718242bbf 100644 --- a/exporter/elasticsearchexporter/esclient.go +++ b/exporter/elasticsearchexporter/esclient.go @@ -90,16 +90,6 @@ func newElasticsearchClient( headers := make(http.Header) headers.Set("User-Agent", userAgent) - // maxRetries configures the maximum number of event publishing attempts, - // including the first send and additional retries. - - maxRetries := config.Retry.MaxRequests - 1 - retryDisabled := !config.Retry.Enabled || maxRetries <= 0 - - if retryDisabled { - maxRetries = 0 - } - // endpoints converts Config.Endpoints, Config.CloudID, // and Config.ClientConfig.Endpoint to a list of addresses. endpoints, err := config.endpoints() @@ -113,6 +103,11 @@ func newElasticsearchClient( logResponseBody: config.LogResponseBody, } + maxRetries := defaultMaxRetries + if config.Retry.MaxRetries != 0 { + maxRetries = config.Retry.MaxRetries + } + return elasticsearch.NewClient(elasticsearch.Config{ Transport: httpClient.Transport, @@ -125,7 +120,7 @@ func newElasticsearchClient( // configure retry behavior RetryOnStatus: config.Retry.RetryOnStatus, - DisableRetry: retryDisabled, + DisableRetry: !config.Retry.Enabled, EnableRetryOnTimeout: config.Retry.Enabled, //RetryOnError: retryOnError, // should be used from esclient version 8 onwards MaxRetries: maxRetries, diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index f1b455e41e1e..9a9b86a5be6f 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -540,14 +540,10 @@ func TestExporterLogs(t *testing.T) { t.Run("no retry", func(t *testing.T) { configurations := map[string]func(*Config){ - "max_requests limited": func(cfg *Config) { - cfg.Retry.MaxRequests = 1 - cfg.Retry.InitialInterval = 1 * time.Millisecond - cfg.Retry.MaxInterval = 10 * time.Millisecond - }, "retry.enabled is false": func(cfg *Config) { cfg.Retry.Enabled = false - cfg.Retry.MaxRequests = 10 + cfg.Retry.RetryOnStatus = []int{429} + cfg.Retry.MaxRetries = 10 cfg.Retry.InitialInterval = 1 * time.Millisecond cfg.Retry.MaxInterval = 10 * time.Millisecond }, @@ -557,7 +553,7 @@ func TestExporterLogs(t *testing.T) { "fail http request": func(attempts *atomic.Int64) bulkHandler { return func([]itemRequest) ([]itemResponse, error) { attempts.Add(1) - return nil, &httpTestError{message: "oops"} + return nil, &httpTestError{message: "oops", status: 429} } }, "fail item": func(attempts *atomic.Int64) bulkHandler { diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 3f48ca1e2ec7..61af38d5cee6 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -63,7 +63,7 @@ func createDefaultConfig() component.Config { }, Retry: RetrySettings{ Enabled: true, - MaxRequests: 3, + MaxRetries: 0, // default is set in exporter code InitialInterval: 100 * time.Millisecond, MaxInterval: 1 * time.Minute, RetryOnStatus: []int{ @@ -110,7 +110,7 @@ func createLogsExporter( set.Logger.Warn("index option are deprecated and replaced with logs_index and traces_index.") index = cf.Index } - logConfigDeprecationWarnings(cf, set.Logger) + handleDeprecatedConfig(cf, set.Logger) exporter := newExporter(cf, set, index, cf.LogsDynamicIndex.Enabled) @@ -129,7 +129,7 @@ func createMetricsExporter( cfg component.Config, ) (exporter.Metrics, error) { cf := cfg.(*Config) - logConfigDeprecationWarnings(cf, set.Logger) + handleDeprecatedConfig(cf, set.Logger) exporter := newExporter(cf, set, cf.MetricsIndex, cf.MetricsDynamicIndex.Enabled) @@ -147,7 +147,7 @@ func createTracesExporter(ctx context.Context, cfg component.Config, ) (exporter.Traces, error) { cf := cfg.(*Config) - logConfigDeprecationWarnings(cf, set.Logger) + handleDeprecatedConfig(cf, set.Logger) exporter := newExporter(cf, set, cf.TracesIndex, cf.TracesDynamicIndex.Enabled) diff --git a/exporter/elasticsearchexporter/testdata/config.yaml b/exporter/elasticsearchexporter/testdata/config.yaml index 6f614399b579..d76d300a51c1 100644 --- a/exporter/elasticsearchexporter/testdata/config.yaml +++ b/exporter/elasticsearchexporter/testdata/config.yaml @@ -17,7 +17,7 @@ elasticsearch/trace: flush: bytes: 10485760 retry: - max_requests: 5 + max_retries: 5 retry_on_status: - 429 - 500 @@ -38,7 +38,7 @@ elasticsearch/metric: flush: bytes: 10485760 retry: - max_requests: 5 + max_retries: 5 retry_on_status: - 429 - 500 @@ -61,7 +61,7 @@ elasticsearch/log: flush: bytes: 10485760 retry: - max_requests: 5 + max_retries: 5 retry_on_status: - 429 - 500 From 2c349181a2e7f0eab0f115f92e15753a1fffe787 Mon Sep 17 00:00:00 2001 From: Yifan Yang Date: Fri, 18 Oct 2024 06:12:38 +0800 Subject: [PATCH 08/21] [chore][docs][pkg/stanza] Fix typo in field.md link to entry.md (#35854) #### Description Fixed a typo in the pkg/stanza/docs/types/field.md documentation. The link was incorrectly pointing to ../types/field.md and is now corrected to ../types/entry.md. #### Testing #### Documentation Signed-off-by: YifanYang6 --- pkg/stanza/docs/types/field.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/stanza/docs/types/field.md b/pkg/stanza/docs/types/field.md index 35f54be3bc9b..b6fb313b8c6d 100644 --- a/pkg/stanza/docs/types/field.md +++ b/pkg/stanza/docs/types/field.md @@ -1,6 +1,6 @@ ## Fields -A _Field_ is a reference to a value in a log [entry](../types/field.md). +A _Field_ is a reference to a value in a log [entry](../types/entry.md). Many [operators](../operators/README.md) use fields in their configurations. For example, parsers use fields to specify which value to parse and where to write a new value. From d5c641a07c2bca9d5802ab61710bb72dc3ab2f9e Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Fri, 18 Oct 2024 05:34:21 +0200 Subject: [PATCH 09/21] [exporter/opensearch] chore: remove redundant config Validate call (#35233) **Description:** Configuration validation is done during collector's startup, making it redundant when being called inside component's logic. This PR removes the Validate call done during exporter's constructor. **Link to tracking Issue:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33498 (Last component, will close the issue) **Testing:** Added default config use case (validate error) **Documentation:** --- exporter/opensearchexporter/config_test.go | 7 ++++++ exporter/opensearchexporter/factory.go | 16 +++++-------- exporter/opensearchexporter/factory_test.go | 24 ------------------- .../opensearchexporter/sso_log_exporter.go | 8 ++----- .../opensearchexporter/sso_trace_exporter.go | 8 ++----- .../opensearchexporter/testdata/config.yaml | 2 ++ 6 files changed, 19 insertions(+), 46 deletions(-) diff --git a/exporter/opensearchexporter/config_test.go b/exporter/opensearchexporter/config_test.go index d6840945c588..921ce1a00428 100644 --- a/exporter/opensearchexporter/config_test.go +++ b/exporter/opensearchexporter/config_test.go @@ -44,6 +44,13 @@ func TestLoadConfig(t *testing.T) { expected: sampleCfg, configValidateAssert: assert.NoError, }, + { + id: component.NewIDWithName(metadata.Type, "default"), + expected: withDefaultConfig(), + configValidateAssert: func(t assert.TestingT, err error, _ ...any) bool { + return assert.ErrorContains(t, err, "endpoint must be specified") + }, + }, { id: component.NewIDWithName(metadata.Type, "trace"), expected: &Config{ diff --git a/exporter/opensearchexporter/factory.go b/exporter/opensearchexporter/factory.go index a10073ca04ae..418987167a32 100644 --- a/exporter/opensearchexporter/factory.go +++ b/exporter/opensearchexporter/factory.go @@ -41,12 +41,10 @@ func newDefaultConfig() component.Config { func createTracesExporter(ctx context.Context, set exporter.Settings, - cfg component.Config) (exporter.Traces, error) { + cfg component.Config, +) (exporter.Traces, error) { c := cfg.(*Config) - te, e := newSSOTracesExporter(c, set) - if e != nil { - return nil, e - } + te := newSSOTracesExporter(c, set) return exporterhelper.NewTracesExporter(ctx, set, cfg, te.pushTraceData, @@ -58,12 +56,10 @@ func createTracesExporter(ctx context.Context, func createLogsExporter(ctx context.Context, set exporter.Settings, - cfg component.Config) (exporter.Logs, error) { + cfg component.Config, +) (exporter.Logs, error) { c := cfg.(*Config) - le, e := newLogExporter(c, set) - if e != nil { - return nil, e - } + le := newLogExporter(c, set) return exporterhelper.NewLogsExporter(ctx, set, cfg, le.pushLogData, diff --git a/exporter/opensearchexporter/factory_test.go b/exporter/opensearchexporter/factory_test.go index f64dd285231a..1f3ab8ccc31d 100644 --- a/exporter/opensearchexporter/factory_test.go +++ b/exporter/opensearchexporter/factory_test.go @@ -20,30 +20,6 @@ func TestCreateDefaultConfig(t *testing.T) { assert.NoError(t, componenttest.CheckConfigStruct(cfg)) } -func TestFactory_CreateMetricsExporter_Fail(t *testing.T) { - factory := NewFactory() - cfg := factory.CreateDefaultConfig() - params := exportertest.NewNopSettings() - _, err := factory.CreateMetricsExporter(context.Background(), params, cfg) - require.Error(t, err, "expected an error when creating a metrics exporter") -} - -func TestFactory_CreateTracesExporter_Fail(t *testing.T) { - factory := NewFactory() - cfg := factory.CreateDefaultConfig() - params := exportertest.NewNopSettings() - _, err := factory.CreateTracesExporter(context.Background(), params, cfg) - require.Error(t, err, "expected an error when creating a traces exporter") -} - -func TestFactory_CreateLogsExporter_Fail(t *testing.T) { - factory := NewFactory() - cfg := factory.CreateDefaultConfig() - params := exportertest.NewNopSettings() - _, err := factory.CreateLogsExporter(context.Background(), params, cfg) - require.Error(t, err, "expected an error when creating a logs exporter") -} - func TestFactory_CreateTracesExporter(t *testing.T) { factory := NewFactory() cfg := withDefaultConfig(func(cfg *Config) { diff --git a/exporter/opensearchexporter/sso_log_exporter.go b/exporter/opensearchexporter/sso_log_exporter.go index bfa34d90d1f1..fe3584f3e0f0 100644 --- a/exporter/opensearchexporter/sso_log_exporter.go +++ b/exporter/opensearchexporter/sso_log_exporter.go @@ -23,11 +23,7 @@ type logExporter struct { telemetry component.TelemetrySettings } -func newLogExporter(cfg *Config, set exporter.Settings) (*logExporter, error) { - if err := cfg.Validate(); err != nil { - return nil, err - } - +func newLogExporter(cfg *Config, set exporter.Settings) *logExporter { model := &encodeModel{ dedup: cfg.Dedup, dedot: cfg.Dedot, @@ -45,7 +41,7 @@ func newLogExporter(cfg *Config, set exporter.Settings) (*logExporter, error) { bulkAction: cfg.BulkAction, httpSettings: cfg.ClientConfig, model: model, - }, nil + } } func (l *logExporter) Start(ctx context.Context, host component.Host) error { diff --git a/exporter/opensearchexporter/sso_trace_exporter.go b/exporter/opensearchexporter/sso_trace_exporter.go index fdc6dab67e87..3c0f2e4b2211 100644 --- a/exporter/opensearchexporter/sso_trace_exporter.go +++ b/exporter/opensearchexporter/sso_trace_exporter.go @@ -25,11 +25,7 @@ type ssoTracesExporter struct { telemetry component.TelemetrySettings } -func newSSOTracesExporter(cfg *Config, set exporter.Settings) (*ssoTracesExporter, error) { - if err := cfg.Validate(); err != nil { - return nil, err - } - +func newSSOTracesExporter(cfg *Config, set exporter.Settings) *ssoTracesExporter { model := &encodeModel{ dataset: cfg.Dataset, namespace: cfg.Namespace, @@ -42,7 +38,7 @@ func newSSOTracesExporter(cfg *Config, set exporter.Settings) (*ssoTracesExporte bulkAction: cfg.BulkAction, model: model, httpSettings: cfg.ClientConfig, - }, nil + } } func (s *ssoTracesExporter) Start(ctx context.Context, host component.Host) error { diff --git a/exporter/opensearchexporter/testdata/config.yaml b/exporter/opensearchexporter/testdata/config.yaml index a187af23318e..2f967002ed96 100644 --- a/exporter/opensearchexporter/testdata/config.yaml +++ b/exporter/opensearchexporter/testdata/config.yaml @@ -7,6 +7,8 @@ opensearch: http: endpoint: https://opensearch.example.com:9200 +opensearch/default: + opensearch/empty_namespace: dataset: ngnix namespace: "" From 17afe149c35348922df127d2e9263b60236ec501 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 17 Oct 2024 22:53:05 -0700 Subject: [PATCH 10/21] [receiver/ntp] add initial implementation (#35850) #### Description Adds initial implementation of ntpreceiver. #### Link to tracking issue #34375 --- receiver/ntpreceiver/config.go | 19 ++++++ receiver/ntpreceiver/config_test.go | 65 +++++++++++++++++++ receiver/ntpreceiver/documentation.md | 4 +- receiver/ntpreceiver/factory.go | 24 ++++++- receiver/ntpreceiver/factory_test.go | 18 +++++ .../ntpreceiver/generated_component_test.go | 51 +++++++++++++++ receiver/ntpreceiver/go.mod | 3 +- receiver/ntpreceiver/go.sum | 2 + .../internal/metadata/generated_metrics.go | 4 +- .../metadata/generated_metrics_test.go | 4 +- receiver/ntpreceiver/metadata.yaml | 8 +-- receiver/ntpreceiver/receiver.go | 46 +++++++++++++ 12 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 receiver/ntpreceiver/config_test.go create mode 100644 receiver/ntpreceiver/factory_test.go create mode 100644 receiver/ntpreceiver/receiver.go diff --git a/receiver/ntpreceiver/config.go b/receiver/ntpreceiver/config.go index b5f78740c498..934d3df77021 100644 --- a/receiver/ntpreceiver/config.go +++ b/receiver/ntpreceiver/config.go @@ -4,6 +4,11 @@ package ntpreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver" import ( + "errors" + "fmt" + "net" + "time" + "go.opentelemetry.io/collector/receiver/scraperhelper" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver/internal/metadata" @@ -13,5 +18,19 @@ import ( type Config struct { scraperhelper.ControllerConfig `mapstructure:",squash"` metadata.MetricsBuilderConfig `mapstructure:",squash"` + Version int `mapstructure:"version"` Endpoint string `mapstructure:"endpoint"` } + +func (c *Config) Validate() error { + var errs []error + _, _, err := net.SplitHostPort(c.Endpoint) + if err != nil { + errs = append(errs, err) + } + // respect terms of service https://www.pool.ntp.org/tos.html + if c.ControllerConfig.CollectionInterval < 30*time.Minute { + errs = append(errs, fmt.Errorf("collection interval %v is less than minimum 30m", c.ControllerConfig.CollectionInterval)) + } + return errors.Join(errs...) +} diff --git a/receiver/ntpreceiver/config_test.go b/receiver/ntpreceiver/config_test.go new file mode 100644 index 000000000000..1ba68eec7f26 --- /dev/null +++ b/receiver/ntpreceiver/config_test.go @@ -0,0 +1,65 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ntpreceiver + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/receiver/scraperhelper" +) + +func TestValidate(t *testing.T) { + for _, tt := range []struct { + name string + c *Config + errorExpected string + }{ + { + name: "no host", + c: &Config{ + Version: 4, + Endpoint: "", + ControllerConfig: scraperhelper.ControllerConfig{CollectionInterval: 45 * time.Minute}, + }, + errorExpected: "missing port in address", + }, + { + name: "no port", + c: &Config{ + Version: 4, + Endpoint: "pool.ntp.org", + ControllerConfig: scraperhelper.ControllerConfig{CollectionInterval: 45 * time.Minute}, + }, + errorExpected: "address pool.ntp.org: missing port in address", + }, + { + name: "valid", + c: &Config{ + Version: 4, + Endpoint: "pool.ntp.org:123", + ControllerConfig: scraperhelper.ControllerConfig{CollectionInterval: 45 * time.Minute}, + }, + }, + { + name: "interval too small", + c: &Config{ + Version: 4, + Endpoint: "pool.ntp.org:123", + ControllerConfig: scraperhelper.ControllerConfig{CollectionInterval: 29 * time.Minute}, + }, + errorExpected: "collection interval 29m0s is less than minimum 30m", + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := tt.c.Validate() + if tt.errorExpected == "" { + require.NoError(t, err) + } else { + require.EqualError(t, err, tt.errorExpected) + } + }) + } +} diff --git a/receiver/ntpreceiver/documentation.md b/receiver/ntpreceiver/documentation.md index 182193963883..01463c1de0cf 100644 --- a/receiver/ntpreceiver/documentation.md +++ b/receiver/ntpreceiver/documentation.md @@ -14,11 +14,11 @@ metrics: ### ntp.offset -Time difference between local and NTP server clocks in seconds. +Time difference between local and NTP server clocks | Unit | Metric Type | Value Type | | ---- | ----------- | ---------- | -| s | Gauge | Int | +| ns | Gauge | Int | ## Resource Attributes diff --git a/receiver/ntpreceiver/factory.go b/receiver/ntpreceiver/factory.go index 07b3a9a7534f..7ff2d5a4be22 100644 --- a/receiver/ntpreceiver/factory.go +++ b/receiver/ntpreceiver/factory.go @@ -5,6 +5,7 @@ package ntpreceiver // import "github.com/open-telemetry/opentelemetry-collector import ( "context" + "time" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" @@ -23,12 +24,29 @@ func NewFactory() receiver.Factory { } func createDefaultConfig() component.Config { + scraperConfig := scraperhelper.NewDefaultControllerConfig() + scraperConfig.CollectionInterval = 30 * time.Minute return &Config{ - ControllerConfig: scraperhelper.NewDefaultControllerConfig(), + ControllerConfig: scraperConfig, MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(), + Version: 4, + Endpoint: "pool.ntp.org:123", } } -func createMetricsReceiver(_ context.Context, _ receiver.Settings, _ component.Config, _ consumer.Metrics) (receiver.Metrics, error) { - return nil, nil +func createMetricsReceiver(_ context.Context, settings receiver.Settings, cfg component.Config, consumer consumer.Metrics) (receiver.Metrics, error) { + rCfg := cfg.(*Config) + mp := newScraper(rCfg, settings) + s, err := scraperhelper.NewScraper(metadata.Type, mp.scrape) + if err != nil { + return nil, err + } + opt := scraperhelper.AddScraper(s) + + return scraperhelper.NewScraperControllerReceiver( + &rCfg.ControllerConfig, + settings, + consumer, + opt, + ) } diff --git a/receiver/ntpreceiver/factory_test.go b/receiver/ntpreceiver/factory_test.go new file mode 100644 index 000000000000..718ced896c44 --- /dev/null +++ b/receiver/ntpreceiver/factory_test.go @@ -0,0 +1,18 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ntpreceiver + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCreateDefaultConfig(t *testing.T) { + c := createDefaultConfig().(*Config) + require.Equal(t, 4, c.Version) + require.Equal(t, "pool.ntp.org:123", c.Endpoint) + require.Equal(t, 30*time.Minute, c.CollectionInterval) +} diff --git a/receiver/ntpreceiver/generated_component_test.go b/receiver/ntpreceiver/generated_component_test.go index 5e8b22347941..a454a725fcfc 100644 --- a/receiver/ntpreceiver/generated_component_test.go +++ b/receiver/ntpreceiver/generated_component_test.go @@ -3,10 +3,16 @@ package ntpreceiver import ( + "context" "testing" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" ) func TestComponentFactoryType(t *testing.T) { @@ -16,3 +22,48 @@ func TestComponentFactoryType(t *testing.T) { func TestComponentConfigStruct(t *testing.T) { require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) } + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + tests := []struct { + name string + createFn func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) + }{ + + { + name: "metrics", + createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateMetrics(ctx, set, cfg, consumertest.NewNop()) + }, + }, + } + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + + for _, tt := range tests { + t.Run(tt.name+"-shutdown", func(t *testing.T) { + c, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run(tt.name+"-lifecycle", func(t *testing.T) { + firstRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + host := componenttest.NewNopHost() + require.NoError(t, err) + require.NoError(t, firstRcvr.Start(context.Background(), host)) + require.NoError(t, firstRcvr.Shutdown(context.Background())) + secondRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(), cfg) + require.NoError(t, err) + require.NoError(t, secondRcvr.Start(context.Background(), host)) + require.NoError(t, secondRcvr.Shutdown(context.Background())) + }) + } +} diff --git a/receiver/ntpreceiver/go.mod b/receiver/ntpreceiver/go.mod index 71d67ef93dd2..2d20374b13e1 100644 --- a/receiver/ntpreceiver/go.mod +++ b/receiver/ntpreceiver/go.mod @@ -3,11 +3,13 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntprec go 1.22.0 require ( + github.com/beevik/ntp v1.4.3 github.com/google/go-cmp v0.6.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae go.opentelemetry.io/collector/confmap v1.17.1-0.20241008154146-ea48c09c31ae go.opentelemetry.io/collector/consumer v0.111.1-0.20241008154146-ea48c09c31ae + go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae go.opentelemetry.io/collector/filter v0.111.1-0.20241008154146-ea48c09c31ae go.opentelemetry.io/collector/pdata v1.17.1-0.20241008154146-ea48c09c31ae go.opentelemetry.io/collector/receiver v0.111.1-0.20241008154146-ea48c09c31ae @@ -34,7 +36,6 @@ require ( github.com/rogpeppe/go-internal v1.12.0 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.111.1-0.20241008154146-ea48c09c31ae // indirect go.opentelemetry.io/collector/consumer/consumerprofiles v0.111.1-0.20241008154146-ea48c09c31ae // indirect - go.opentelemetry.io/collector/consumer/consumertest v0.111.1-0.20241008154146-ea48c09c31ae // indirect go.opentelemetry.io/collector/internal/globalsignal v0.111.1-0.20241008154146-ea48c09c31ae // indirect go.opentelemetry.io/collector/pdata/pprofile v0.111.1-0.20241008154146-ea48c09c31ae // indirect go.opentelemetry.io/collector/pipeline v0.111.1-0.20241008154146-ea48c09c31ae // indirect diff --git a/receiver/ntpreceiver/go.sum b/receiver/ntpreceiver/go.sum index 65f0a94a89a6..7c55d9505af5 100644 --- a/receiver/ntpreceiver/go.sum +++ b/receiver/ntpreceiver/go.sum @@ -1,3 +1,5 @@ +github.com/beevik/ntp v1.4.3 h1:PlbTvE5NNy4QHmA4Mg57n7mcFTmr1W1j3gcK7L1lqho= +github.com/beevik/ntp v1.4.3/go.mod h1:Unr8Zg+2dRn7d8bHFuehIMSvvUYssHMxW3Q5Nx4RW5Q= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/receiver/ntpreceiver/internal/metadata/generated_metrics.go b/receiver/ntpreceiver/internal/metadata/generated_metrics.go index afc023c251d6..0b77e5ddfea6 100644 --- a/receiver/ntpreceiver/internal/metadata/generated_metrics.go +++ b/receiver/ntpreceiver/internal/metadata/generated_metrics.go @@ -21,8 +21,8 @@ type metricNtpOffset struct { // init fills ntp.offset metric with initial data. func (m *metricNtpOffset) init() { m.data.SetName("ntp.offset") - m.data.SetDescription("Time difference between local and NTP server clocks in seconds.") - m.data.SetUnit("s") + m.data.SetDescription("Time difference between local and NTP server clocks") + m.data.SetUnit("ns") m.data.SetEmptyGauge() } diff --git a/receiver/ntpreceiver/internal/metadata/generated_metrics_test.go b/receiver/ntpreceiver/internal/metadata/generated_metrics_test.go index 4ea82d6f71ac..4f92fc0d4c33 100644 --- a/receiver/ntpreceiver/internal/metadata/generated_metrics_test.go +++ b/receiver/ntpreceiver/internal/metadata/generated_metrics_test.go @@ -101,8 +101,8 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["ntp.offset"] = true assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) - assert.Equal(t, "Time difference between local and NTP server clocks in seconds.", ms.At(i).Description()) - assert.Equal(t, "s", ms.At(i).Unit()) + assert.Equal(t, "Time difference between local and NTP server clocks", ms.At(i).Description()) + assert.Equal(t, "ns", ms.At(i).Unit()) dp := ms.At(i).Gauge().DataPoints().At(0) assert.Equal(t, start, dp.StartTimestamp()) assert.Equal(t, ts, dp.Timestamp()) diff --git a/receiver/ntpreceiver/metadata.yaml b/receiver/ntpreceiver/metadata.yaml index c6536b0a05c9..f54de24bafdd 100644 --- a/receiver/ntpreceiver/metadata.yaml +++ b/receiver/ntpreceiver/metadata.yaml @@ -16,12 +16,10 @@ resource_attributes: metrics: ntp.offset: - description: Time difference between local and NTP server clocks in seconds. - unit: "s" + description: Time difference between local and NTP server clocks + unit: "ns" gauge: value_type: int enabled: true -tests: - skip_lifecycle: true - skip_shutdown: true \ No newline at end of file +tests: \ No newline at end of file diff --git a/receiver/ntpreceiver/receiver.go b/receiver/ntpreceiver/receiver.go new file mode 100644 index 000000000000..d52605431b46 --- /dev/null +++ b/receiver/ntpreceiver/receiver.go @@ -0,0 +1,46 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ntpreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver" + +import ( + "context" + "time" + + "github.com/beevik/ntp" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/receiver" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/ntpreceiver/internal/metadata" +) + +type scraper struct { + logger *zap.Logger + mb *metadata.MetricsBuilder + version int + timeout time.Duration + endpoint string +} + +func (s *scraper) scrape(context.Context) (pmetric.Metrics, error) { + options := ntp.QueryOptions{Version: s.version, Timeout: s.timeout} + response, err := ntp.QueryWithOptions(s.endpoint, options) + if err != nil { + return pmetric.Metrics{}, err + } + s.mb.RecordNtpOffsetDataPoint(pcommon.NewTimestampFromTime(time.Now()), response.ClockOffset.Nanoseconds()) + s.mb.NewResourceBuilder().SetNtpHost(s.endpoint) + return s.mb.Emit(), nil +} + +func newScraper(cfg *Config, settings receiver.Settings) *scraper { + return &scraper{ + logger: settings.TelemetrySettings.Logger, + mb: metadata.NewMetricsBuilder(cfg.MetricsBuilderConfig, settings), + version: cfg.Version, + timeout: cfg.ControllerConfig.Timeout, + endpoint: cfg.Endpoint, + } +} From 1fab9bbe819594e0b2ae50ae9e6aed057031f7e5 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 18 Oct 2024 07:18:24 +0100 Subject: [PATCH 11/21] [exporter/elasticsearch] Make OTel mapping mode send to data streams only (#35839) #### Description Make OTel mapping mode use RequireDataStream in docappender, which means it will only send to data streams. This prevents auto creating regular indices in OTel mapping mode due to a race condition in Elasticsearch where otel-data index templates are not ready. #### Link to tracking issue #### Testing #### Documentation --- ...xporter_otel-mode-require-data-stream.yaml | 27 +++++++++ exporter/elasticsearchexporter/bulkindexer.go | 1 + .../elasticsearchexporter/bulkindexer_test.go | 55 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 .chloggen/elasticsearchexporter_otel-mode-require-data-stream.yaml diff --git a/.chloggen/elasticsearchexporter_otel-mode-require-data-stream.yaml b/.chloggen/elasticsearchexporter_otel-mode-require-data-stream.yaml new file mode 100644 index 000000000000..e7aebbbff1b9 --- /dev/null +++ b/.chloggen/elasticsearchexporter_otel-mode-require-data-stream.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Make OTel mapping mode send to data streams only + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35839] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: This prevents auto creating regular indices in OTel mapping mode due to a race condition in Elasticsearch where otel-data index templates are not ready. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/bulkindexer.go b/exporter/elasticsearchexporter/bulkindexer.go index 62bc329a26f8..21b48814914d 100644 --- a/exporter/elasticsearchexporter/bulkindexer.go +++ b/exporter/elasticsearchexporter/bulkindexer.go @@ -73,6 +73,7 @@ func bulkIndexerConfig(client *elasticsearch.Client, config *Config) docappender MaxDocumentRetries: maxDocRetries, Pipeline: config.Pipeline, RetryOnDocumentStatus: config.Retry.RetryOnStatus, + RequireDataStream: config.MappingMode() == MappingOTel, } } diff --git a/exporter/elasticsearchexporter/bulkindexer_test.go b/exporter/elasticsearchexporter/bulkindexer_test.go index b417942734d6..7a75c6f5a0f1 100644 --- a/exporter/elasticsearchexporter/bulkindexer_test.go +++ b/exporter/elasticsearchexporter/bulkindexer_test.go @@ -115,6 +115,61 @@ func TestAsyncBulkIndexer_flush(t *testing.T) { } } +func TestAsyncBulkIndexer_requireDataStream(t *testing.T) { + tests := []struct { + name string + config Config + wantRequireDataStream bool + }{ + { + name: "ecs", + config: Config{ + NumWorkers: 1, + Mapping: MappingsSettings{Mode: MappingECS.String()}, + }, + wantRequireDataStream: false, + }, + { + name: "otel", + config: Config{ + NumWorkers: 1, + Mapping: MappingsSettings{Mode: MappingOTel.String()}, + }, + wantRequireDataStream: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + requireDataStreamCh := make(chan bool, 1) + client, err := elasticsearch.NewClient(elasticsearch.Config{Transport: &mockTransport{ + RoundTripFunc: func(r *http.Request) (*http.Response, error) { + if r.URL.Path == "/_bulk" { + requireDataStreamCh <- r.URL.Query().Get("require_data_stream") == "true" + } + return &http.Response{ + Header: http.Header{"X-Elastic-Product": []string{"Elasticsearch"}}, + Body: io.NopCloser(strings.NewReader(successResp)), + }, nil + }, + }}) + require.NoError(t, err) + + bulkIndexer, err := newAsyncBulkIndexer(zap.NewNop(), client, &tt.config) + require.NoError(t, err) + session, err := bulkIndexer.StartSession(context.Background()) + require.NoError(t, err) + + assert.NoError(t, session.Add(context.Background(), "foo", strings.NewReader(`{"foo": "bar"}`), nil)) + assert.NoError(t, bulkIndexer.Close(context.Background())) + + assert.Equal(t, tt.wantRequireDataStream, <-requireDataStreamCh) + }) + } +} + func TestAsyncBulkIndexer_flush_error(t *testing.T) { tests := []struct { name string From e6936f2736b8df672545ee9278f1c61517a911e4 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:03:41 +0530 Subject: [PATCH 12/21] [chore][receiver/loki] follow receiver contract (#35327) **Description:** Follow receiver contract for `loki`. This also includes an internal errorutil package which will be used by other network receivers as well. **Link to tracking Issue:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/5909 **Testing:** Added --- internal/coreinternal/errorutil/grpc.go | 25 ++++++++ internal/coreinternal/go.mod | 2 +- receiver/lokireceiver/go.mod | 2 +- receiver/lokireceiver/loki.go | 8 +++ receiver/lokireceiver/loki_test.go | 76 +++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 internal/coreinternal/errorutil/grpc.go diff --git a/internal/coreinternal/errorutil/grpc.go b/internal/coreinternal/errorutil/grpc.go new file mode 100644 index 000000000000..08b75990f0fc --- /dev/null +++ b/internal/coreinternal/errorutil/grpc.go @@ -0,0 +1,25 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package errorutil // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/errorutil" + +import ( + "go.opentelemetry.io/collector/consumer/consumererror" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func GrpcError(err error) error { + s, ok := status.FromError(err) + if !ok { + // Default to a retryable error + // https://github.com/open-telemetry/opentelemetry-proto/blob/main/docs/specification.md#failures + code := codes.Unavailable + if consumererror.IsPermanent(err) { + // non-retryable error + code = codes.Unknown + } + s = status.New(code, err.Error()) + } + return s.Err() +} diff --git a/internal/coreinternal/go.mod b/internal/coreinternal/go.mod index ed4f490e9bcb..8675c13635fe 100644 --- a/internal/coreinternal/go.mod +++ b/internal/coreinternal/go.mod @@ -23,6 +23,7 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/text v0.19.0 + google.golang.org/grpc v1.67.1 ) require ( @@ -87,7 +88,6 @@ require ( golang.org/x/sys v0.25.0 // indirect golang.org/x/tools v0.23.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect - google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/receiver/lokireceiver/go.mod b/receiver/lokireceiver/go.mod index 1c5b291f8fc3..d5be9e59e979 100644 --- a/receiver/lokireceiver/go.mod +++ b/receiver/lokireceiver/go.mod @@ -9,7 +9,7 @@ require ( github.com/grafana/loki/pkg/push v0.0.0-20240514112848-a1b1eeb09583 github.com/json-iterator/go v1.1.12 github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.111.0 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.111.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.111.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.111.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.111.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/loki v0.111.0 diff --git a/receiver/lokireceiver/loki.go b/receiver/lokireceiver/loki.go index 1d4cc5c3f0c1..a994c0ae121e 100644 --- a/receiver/lokireceiver/loki.go +++ b/receiver/lokireceiver/loki.go @@ -21,6 +21,7 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/errorutil" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/loki" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/lokireceiver/internal" ) @@ -163,6 +164,9 @@ func (r *lokiReceiver) Push(ctx context.Context, pushRequest *push.PushRequest) logRecordCount := logs.LogRecordCount() err = r.nextConsumer.ConsumeLogs(ctx, logs) r.obsrepGRPC.EndLogsOp(ctx, "protobuf", logRecordCount, err) + if err != nil { + return &push.PushResponse{}, errorutil.GrpcError(err) + } return &push.PushResponse{}, nil } @@ -219,6 +223,10 @@ func handleLogs(resp http.ResponseWriter, req *http.Request, r *lokiReceiver) { logRecordCount := logs.LogRecordCount() err = r.nextConsumer.ConsumeLogs(ctx, logs) r.obsrepHTTP.EndLogsOp(ctx, "json", logRecordCount, err) + if err != nil { + errorutil.HTTPError(resp, err) + return + } resp.WriteHeader(http.StatusNoContent) } diff --git a/receiver/lokireceiver/loki_test.go b/receiver/lokireceiver/loki_test.go index bf208b780d39..1b17bdc11314 100644 --- a/receiver/lokireceiver/loki_test.go +++ b/receiver/lokireceiver/loki_test.go @@ -8,6 +8,7 @@ import ( "compress/gzip" "compress/zlib" "context" + "errors" "fmt" "net" "net/http" @@ -23,6 +24,7 @@ import ( "go.opentelemetry.io/collector/config/configgrpc" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/confignet" + "go.opentelemetry.io/collector/consumer/consumererror" "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" @@ -362,6 +364,80 @@ func TestSendingPushRequestToGRPCEndpoint(t *testing.T) { } } +func TestExpectedStatus(t *testing.T) { + + testcases := []struct { + name string + err error + expectedGrpcError string + expectedHTTPError string + }{ + { + name: "permanent-error", + err: consumererror.NewPermanent(errors.New("permanent")), + expectedGrpcError: "rpc error: code = Unknown desc = Permanent error: permanent", + expectedHTTPError: "failed to upload logs; HTTP status code: 400", + }, + { + name: "non-permanent-error", + err: errors.New("non-permanent"), + expectedGrpcError: "rpc error: code = Unavailable desc = non-permanent", + expectedHTTPError: "failed to upload logs; HTTP status code: 503", + }, + } + for _, tt := range testcases { + t.Run(tt.name, func(t *testing.T) { + httpAddr := testutil.GetAvailableLocalAddress(t) + config := &Config{ + Protocols: Protocols{ + GRPC: &configgrpc.ServerConfig{ + NetAddr: confignet.AddrConfig{ + Endpoint: testutil.GetAvailableLocalAddress(t), + Transport: confignet.TransportTypeTCP, + }, + }, + HTTP: &confighttp.ServerConfig{ + Endpoint: httpAddr, + }, + }, + KeepTimestamp: true, + } + + consumer := consumertest.NewErr(tt.err) + lr, err := newLokiReceiver(config, consumer, receivertest.NewNopSettings()) + require.NoError(t, err) + + require.NoError(t, lr.Start(context.Background(), componenttest.NewNopHost())) + t.Cleanup(func() { require.NoError(t, lr.Shutdown(context.Background())) }) + conn, err := grpc.NewClient(config.GRPC.NetAddr.Endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + grpcClient := push.NewPusherClient(conn) + + body := &push.PushRequest{ + Streams: []push.Stream{ + { + Labels: "{foo=\"bar\"}", + Entries: []push.Entry{ + { + Timestamp: time.Unix(0, 1676888496000000000), + Line: "logline 1", + }, + }, + }, + }, + } + + _, err = grpcClient.Push(context.Background(), body) + require.EqualError(t, err, tt.expectedGrpcError) + + _, port, _ := net.SplitHostPort(httpAddr) + collectorAddr := fmt.Sprintf("http://localhost:%s/loki/api/v1/push", port) + require.EqualError(t, sendToCollector(collectorAddr, "application/json", "", []byte(`{"streams": [{"stream": {"foo": "bar"},"values": [[ "1676888496000000000", "logline 1" ]]}]}`)), tt.expectedHTTPError) + }) + } +} + type Log struct { Timestamp int64 Body pcommon.Value From 2b9ec0282937fc7aea8db948fe794a1273f322c5 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Fri, 18 Oct 2024 13:35:41 +0200 Subject: [PATCH 13/21] [chore][receiver/filelog] docs: fix docs on setting `header` to `false` (#35864) Setting `heaader` to `false` results in error when starting the collector. The value of `header` must either be a map or `null`. --- receiver/filelogreceiver/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/receiver/filelogreceiver/README.md b/receiver/filelogreceiver/README.md index c6c4a5fe9c4a..c34df2508a0d 100644 --- a/receiver/filelogreceiver/README.md +++ b/receiver/filelogreceiver/README.md @@ -45,7 +45,7 @@ Tails and parses logs from files. | `resource` | {} | A map of `key: value` pairs to add to the entry's resource. | | `operators` | [] | An array of [operators](../../pkg/stanza/docs/operators/README.md#what-operators-are-available). See below for more details. | | `storage` | none | The ID of a storage extension to be used to store file offsets. File offsets allow the receiver to pick up where it left off in the case of a collector restart. If no storage extension is used, the receiver will manage offsets in memory only. | -| `header` | nil | Specifies options for parsing header metadata. Requires that the `filelog.allowHeaderMetadataParsing` feature gate is enabled. See below for details. Must be `false` when `start_at` is set to `end`. | +| `header` | nil | Specifies options for parsing header metadata. Requires that the `filelog.allowHeaderMetadataParsing` feature gate is enabled. See below for details. Must not be set when `start_at` is set to `end`. | | `header.pattern` | required for header metadata parsing | A regex that matches every header line. | | `header.metadata_operators` | required for header metadata parsing | A list of operators used to parse metadata from the header. | | `retry_on_failure.enabled` | `false` | If `true`, the receiver will pause reading a file and attempt to resend the current batch of logs if it encounters an error from downstream components. | From 633ed51c3ca552c255130e2387ea7382433fd61e Mon Sep 17 00:00:00 2001 From: sh0rez Date: Fri, 18 Oct 2024 17:32:52 +0200 Subject: [PATCH 14/21] processor/deltatocumulative: golden tests (#35562) **Description:** Rewrites most tests to use a pkg/golden like approach: - tests are directories under `testdata/` - tests consist of multiple stages, ran in series - each stage is a [txtar](https://pkg.go.dev/golang.org/x/tools/txtar), containing: - `in`: pmetric yaml of the `ConsumeMetrics` input - `out`: expected output on the sink after calling `ConsumeMetrics` The multi-stage setup allows to exercise multiple Requests in one test. Using txtar allows to co-locate common yaml easily. I plan to add metric assertions in there later too. **Link to tracking Issue:** none **Testing:** Tests were rewritten **Documentation:** not needed --- processor/deltatocumulativeprocessor/go.mod | 8 +- processor/deltatocumulativeprocessor/go.sum | 10 +- .../internal/testar/decode.go | 112 ++++++ .../internal/testar/read_test.go | 58 +++ .../internal/testdata/random/random.go | 24 +- .../deltatocumulativeprocessor/linear.go | 2 +- .../processor_test.go | 336 ++++------------ .../testdata/limit/1.test | 47 +++ .../testdata/limit/2.test | 49 +++ .../testdata/limit/config.yaml | 1 + .../testdata/notemporality-ignored/1.test | 57 +++ .../testdata/notemporality-ignored/in.yaml | 27 -- .../testdata/notemporality-ignored/out.yaml | 27 -- .../testdata/timestamps/1.test | 36 ++ .../testdata/tracking/1.test | 376 ++++++++++++++++++ 15 files changed, 841 insertions(+), 329 deletions(-) create mode 100644 processor/deltatocumulativeprocessor/internal/testar/decode.go create mode 100644 processor/deltatocumulativeprocessor/internal/testar/read_test.go create mode 100644 processor/deltatocumulativeprocessor/testdata/limit/1.test create mode 100644 processor/deltatocumulativeprocessor/testdata/limit/2.test create mode 100644 processor/deltatocumulativeprocessor/testdata/limit/config.yaml create mode 100644 processor/deltatocumulativeprocessor/testdata/notemporality-ignored/1.test delete mode 100644 processor/deltatocumulativeprocessor/testdata/notemporality-ignored/in.yaml delete mode 100644 processor/deltatocumulativeprocessor/testdata/notemporality-ignored/out.yaml create mode 100644 processor/deltatocumulativeprocessor/testdata/timestamps/1.test create mode 100644 processor/deltatocumulativeprocessor/testdata/tracking/1.test diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod index 1c4e225bd097..f6e8f201aa3e 100644 --- a/processor/deltatocumulativeprocessor/go.mod +++ b/processor/deltatocumulativeprocessor/go.mod @@ -5,7 +5,6 @@ go 1.22.0 require ( github.com/google/go-cmp v0.6.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.111.0 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.111.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.111.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae @@ -21,6 +20,8 @@ require ( go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 + golang.org/x/tools v0.25.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -50,13 +51,12 @@ require ( go.opentelemetry.io/collector/processor/processorprofiles v0.111.1-0.20241008154146-ea48c09c31ae // indirect go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/text v0.18.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum index 0b4ad58ccdf4..a60aff0b0442 100644 --- a/processor/deltatocumulativeprocessor/go.sum +++ b/processor/deltatocumulativeprocessor/go.sum @@ -103,8 +103,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -115,12 +115,14 @@ golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/processor/deltatocumulativeprocessor/internal/testar/decode.go b/processor/deltatocumulativeprocessor/internal/testar/decode.go new file mode 100644 index 000000000000..5141df958fbd --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/testar/decode.go @@ -0,0 +1,112 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// testar is a textual archive (based on [golang.org/x/tools/txtar]) to define +// test fixtures. +// +// Archive data is read into struct fields, optionally calling parsers for field +// types other than string or []byte: +// +// type T struct { +// Literal string `testar:"file1"` +// Parsed int `testar:"file2,myparser"` +// } +// +// var into T +// err := Read(data, &into) +// +// See [Read] and [Parser] for examples. +package testar // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/testar" + +import ( + "fmt" + "io/fs" + "reflect" + "strings" + + "golang.org/x/tools/txtar" +) + +// Read archive data into the fields of struct *T +func Read[T any](data []byte, into *T, parsers ...Format) error { + ar := txtar.Parse(data) + return Decode(ar, into, parsers...) +} + +func ReadFile[T any](file string, into *T, parsers ...Format) error { + ar, err := txtar.ParseFile(file) + if err != nil { + return err + } + return Decode(ar, into, parsers...) +} + +func Decode[T any](ar *txtar.Archive, into *T, parsers ...Format) error { + arfs, err := txtar.FS(ar) + if err != nil { + return err + } + + pv := reflect.ValueOf(into) + if pv.Kind() != reflect.Pointer { + return fmt.Errorf("into must be pointer") + } + sv := pv.Elem() + + for i := range sv.NumField() { + f := sv.Type().Field(i) + tag := f.Tag.Get("testar") + if tag == "" { + continue + } + + name, format, _ := strings.Cut(tag, ",") + data, err := fs.ReadFile(arfs, name) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + + err = formats(parsers).Parse(format, data, sv.Field(i).Addr().Interface()) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + } + return nil +} + +type formats []Format + +func (fmts formats) Parse(name string, data []byte, into any) error { + if name == "" { + return LiteralParser(data, into) + } + + for _, f := range fmts { + if f.name == name { + return f.parse(data, into) + } + } + return fmt.Errorf("no such format: %q", name) +} + +type Format struct { + name string + parse func(file []byte, into any) error +} + +func Parser(name string, fn func(data []byte, into any) error) Format { + return Format{name: name, parse: fn} +} + +// LiteralParser sets data unaltered into a []byte or string +func LiteralParser(data []byte, into any) error { + switch ptr := into.(type) { + case *[]byte: + *ptr = append([]byte(nil), data...) + case *string: + *ptr = string(data) + default: + return fmt.Errorf("pass *[]byte, *string or use a parser. got %T", into) + } + return nil +} diff --git a/processor/deltatocumulativeprocessor/internal/testar/read_test.go b/processor/deltatocumulativeprocessor/internal/testar/read_test.go new file mode 100644 index 000000000000..6279ac35527e --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/testar/read_test.go @@ -0,0 +1,58 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testar + +import ( + "fmt" + "strconv" + "strings" +) + +func ExampleRead() { + data := []byte(` +-- foo -- +hello + +-- bar -- +world +`) + + var into struct { + Foo string `testar:"foo"` + Bar []byte `testar:"bar"` + } + + _ = Read(data, &into) + fmt.Printf("foo: %T(%q)\n", into.Foo, into.Foo) + fmt.Printf("bar: %T(%q)\n", into.Bar, into.Bar) + + // Output: + // foo: string("hello\n\n") + // bar: []uint8("world\n") +} + +func ExampleParser() { + data := []byte(` +-- foobar -- +377927 +`) + + var into struct { + Foobar int `testar:"foobar,atoi"` + } + + _ = Read(data, &into, Parser("atoi", func(file []byte, into any) error { + n, err := strconv.Atoi(strings.TrimSpace(string(file))) + if err != nil { + return err + } + *(into.(*int)) = n + return nil + })) + + fmt.Printf("foobar: %T(%d)\n", into.Foobar, into.Foobar) + + // Output: + // foobar: int(377927) +} diff --git a/processor/deltatocumulativeprocessor/internal/testdata/random/random.go b/processor/deltatocumulativeprocessor/internal/testdata/random/random.go index e205fa358882..ca0642cf8795 100644 --- a/processor/deltatocumulativeprocessor/internal/testdata/random/random.go +++ b/processor/deltatocumulativeprocessor/internal/testdata/random/random.go @@ -68,23 +68,35 @@ func (m Metric[P]) Stream() (streams.Ident, P) { } func Resource() pcommon.Resource { + return ResourceN(10) +} + +func ResourceN(n int) pcommon.Resource { res := pcommon.NewResource() - for i := 0; i < 10; i++ { - res.Attributes().PutStr(randStr(), randStr()) - } + Attributes(n).MoveTo(res.Attributes()) return res } func Scope() pcommon.InstrumentationScope { + return ScopeN(3) +} + +func ScopeN(n int) pcommon.InstrumentationScope { scope := pcommon.NewInstrumentationScope() scope.SetName(randStr()) scope.SetVersion(randStr()) - for i := 0; i < 3; i++ { - scope.Attributes().PutStr(randStr(), randStr()) - } + Attributes(n).MoveTo(scope.Attributes()) return scope } +func Attributes(n int) pcommon.Map { + m := pcommon.NewMap() + for i := 0; i < n; i++ { + m.PutStr(randStr(), randStr()) + } + return m +} + func randStr() string { return strconv.FormatInt(randInt(), 16) } diff --git a/processor/deltatocumulativeprocessor/linear.go b/processor/deltatocumulativeprocessor/linear.go index b333ab851627..34edcd377eb2 100644 --- a/processor/deltatocumulativeprocessor/linear.go +++ b/processor/deltatocumulativeprocessor/linear.go @@ -110,7 +110,7 @@ func (p *Linear) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { acc, err := func() (data.Number, error) { if !ok { // new stream: there is no existing aggregation, so start new with current dp - return dp, nil + return dp.Clone(), nil } // tracked stream: add incoming delta dp to existing cumulative aggregation return acc, delta.AccumulateInto(acc, dp) diff --git a/processor/deltatocumulativeprocessor/processor_test.go b/processor/deltatocumulativeprocessor/processor_test.go index a2cbf5957ac6..12d4452e621f 100644 --- a/processor/deltatocumulativeprocessor/processor_test.go +++ b/processor/deltatocumulativeprocessor/processor_test.go @@ -5,308 +5,124 @@ package deltatocumulativeprocessor import ( "context" + "encoding/json" + "errors" + "io/fs" "math" - "math/rand" + "os" "path/filepath" - "strconv" "testing" - "time" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap/confmaptest" "go.opentelemetry.io/collector/consumer/consumertest" - "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/processor" "go.opentelemetry.io/collector/processor/processortest" "go.opentelemetry.io/otel/sdk/metric/metricdata" + "gopkg.in/yaml.v3" - "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics/identity" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data/datatest/compare" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/testdata/random" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/testar" ) -func setup(t *testing.T, cfg *Config) (processor.Metrics, *consumertest.MetricsSink) { - t.Helper() - - next := &consumertest.MetricsSink{} - if cfg == nil { - cfg = &Config{MaxStale: 0, MaxStreams: math.MaxInt} - } - - proc, err := NewFactory().CreateMetrics( - context.Background(), - processortest.NewNopSettings(), - cfg, - next, - ) +func TestProcessor(t *testing.T) { + fis, err := os.ReadDir("testdata") require.NoError(t, err) - return proc, next -} - -// TestAccumulation verifies stream identification works correctly by writing -// 100 random dps spread across 10 different streams. -// Processor output is compared against a manual aggregation on a per-stream basis. -// -// Uses Sum datatype for testing, as we are not testing actual aggregation (see -// internal/data for tests), but proper stream separation -func TestAccumulation(t *testing.T) { - proc, sink := setup(t, nil) - - sum := random.Sum() + for _, fi := range fis { + if !fi.IsDir() { + continue + } - // create 10 distinct streams - const N = 10 - sbs := make([]SumBuilder, N) - for i := range sbs { - _, base := sum.Stream() - sbs[i] = SumBuilder{Metric: sum, base: base} - } + type Stage struct { + In pmetric.Metrics `testar:"in,pmetric"` + Out pmetric.Metrics `testar:"out,pmetric"` + } - // init manual aggregation state - want := make(map[identity.Stream]data.Number) - for _, s := range sbs { - id := s.id(pmetric.AggregationTemporalityCumulative) - want[id] = s.point(0, 0, 0) - } + read := func(file string, into *Stage) error { + return testar.ReadFile(file, into, + testar.Parser("pmetric", unmarshalMetrics), + ) + } - for i := 0; i < 100; i++ { - s := sbs[rand.Intn(N)] + dir := fi.Name() + t.Run(dir, func(t *testing.T) { + file := func(f string) string { + return filepath.Join("testdata", dir, f) + } - v := int64(rand.Intn(255)) - ts := pcommon.Timestamp(i) + ctx := context.Background() + cfg := config(t, file("config.yaml")) + proc, sink := setup(t, cfg) - // write to processor - in := s.delta(s.point(0, ts, v)) - rms := s.resourceMetrics(in) - err := proc.ConsumeMetrics(context.Background(), rms) - require.NoError(t, err) + stages, _ := filepath.Glob(file("*.test")) + for _, file := range stages { + var stage Stage + err := read(file, &stage) + require.NoError(t, err) - // aggregate manually - wantv := want[s.id(pmetric.AggregationTemporalityCumulative)] - wantv.SetIntValue(wantv.IntValue() + v) - wantv.SetTimestamp(ts) - } + sink.Reset() + err = proc.ConsumeMetrics(ctx, stage.In) + require.NoError(t, err) - // get the final processor output for each stream - got := make(map[identity.Stream]data.Number) - for _, md := range sink.AllMetrics() { - metrics.All(md)(func(m metrics.Metric) bool { - sum := metrics.Sum(m) - streams.Datapoints(sum)(func(id identity.Stream, dp data.Number) bool { - got[id] = dp - return true - }) - return true + out := []pmetric.Metrics{stage.Out} + if diff := compare.Diff(out, sink.AllMetrics()); diff != "" { + t.Fatal(diff) + } + } }) - } - sort := cmpopts.SortMaps(func(a, b identity.Stream) bool { - return a.Hash().Sum64() < b.Hash().Sum64() - }) - if diff := compare.Diff(want, got, sort); diff != "" { - t.Fatal(diff) } } -// TestTimestamp verifies timestamp handling, most notably: -// - Timestamp() keeps getting advanced -// - StartTimestamp() stays the same -func TestTimestamps(t *testing.T) { - proc, sink := setup(t, nil) - - sb := stream() - point := func(start, last pcommon.Timestamp) data.Number { - return sb.point(start, last, 0) +func config(t *testing.T, file string) *Config { + cfg := NewFactory().CreateDefaultConfig().(*Config) + cm, err := confmaptest.LoadConf(file) + if errors.Is(err, fs.ErrNotExist) { + return cfg } + require.NoError(t, err) - cases := []struct { - in data.Number - out data.Number - drop bool - }{{ - // first: take as-is - in: point(1000, 1100), - out: point(1000, 1100), - }, { - // subsequent: take, but keep start-ts - in: point(1100, 1200), - out: point(1000, 1200), - }, { - // gap: take - in: point(1300, 1400), - out: point(1000, 1400), - }, { - // out of order - in: point(1200, 1300), - drop: true, - }, { - // older start - in: point(500, 550), - drop: true, - }} - - for i, cs := range cases { - t.Run(strconv.Itoa(i), func(t *testing.T) { - sink.Reset() - - in := sb.resourceMetrics(sb.delta(cs.in)) - want := make([]pmetric.Metrics, 0) - if !cs.drop { - want = []pmetric.Metrics{sb.resourceMetrics(sb.cumul(cs.out))} - } - - err := proc.ConsumeMetrics(context.Background(), in) - require.NoError(t, err) - - out := sink.AllMetrics() - if diff := compare.Diff(want, out); diff != "" { - t.Fatal(diff) - } - }) - } + err = cm.Unmarshal(cfg) + require.NoError(t, err) + return cfg } -func TestStreamLimit(t *testing.T) { - proc, sink := setup(t, &Config{MaxStale: 5 * time.Minute, MaxStreams: 10}) - - good := make([]SumBuilder, 10) - for i := range good { - good[i] = stream() - } - bad := stream() - _ = bad - - diff := func(want, got []pmetric.Metrics) { - t.Helper() - if diff := compare.Diff(want, got); diff != "" { - t.Fatal(diff) - } - } - - writeGood := func(ts pcommon.Timestamp) { - for i, sb := range good { - in := sb.resourceMetrics(sb.delta(sb.point(0, ts+pcommon.Timestamp(i), 0))) - want := sb.resourceMetrics(sb.cumul(sb.point(0, ts+pcommon.Timestamp(i), 0))) - - err := proc.ConsumeMetrics(context.Background(), in) - require.NoError(t, err) +func setup(t *testing.T, cfg *Config) (processor.Metrics, *consumertest.MetricsSink) { + t.Helper() - diff([]pmetric.Metrics{want}, sink.AllMetrics()) - sink.Reset() - } + next := &consumertest.MetricsSink{} + if cfg == nil { + cfg = &Config{MaxStale: 0, MaxStreams: math.MaxInt} } - // write up to limit must work - writeGood(0) - - // extra stream must be dropped, nothing written - in := bad.resourceMetrics(bad.delta(bad.point(0, 0, 0))) - err := proc.ConsumeMetrics(context.Background(), in) + proc, err := NewFactory().CreateMetrics( + context.Background(), + processortest.NewNopSettings(), + cfg, + next, + ) require.NoError(t, err) - diff([]pmetric.Metrics{}, sink.AllMetrics()) - sink.Reset() - - // writing existing streams must still work - writeGood(100) -} - -type copyable interface { - CopyTo(pmetric.Metric) -} - -func (s SumBuilder) resourceMetrics(metrics ...copyable) pmetric.Metrics { - md := pmetric.NewMetrics() - - rm := md.ResourceMetrics().AppendEmpty() - s.Resource().CopyTo(rm.Resource()) - - sm := rm.ScopeMetrics().AppendEmpty() - s.Scope().CopyTo(sm.Scope()) - - for _, m := range metrics { - m.CopyTo(sm.Metrics().AppendEmpty()) - } - return md -} -type SumBuilder struct { - random.Metric[data.Number] - base data.Number + return proc, next } -func (s SumBuilder) with(dps ...data.Number) pmetric.Metric { - m := pmetric.NewMetric() - s.Metric.CopyTo(m) - - for _, dp := range dps { - dp.NumberDataPoint.CopyTo(m.Sum().DataPoints().AppendEmpty()) +func unmarshalMetrics(data []byte, into any) error { + var tmp any + if err := yaml.Unmarshal(data, &tmp); err != nil { + return err } - - return m -} - -func (s SumBuilder) delta(dps ...data.Number) pmetric.Metric { - m := s.with(dps...) - m.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityDelta) - return m -} - -func (s SumBuilder) cumul(dps ...data.Number) pmetric.Metric { - m := s.with(dps...) - m.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - return m -} - -func (s SumBuilder) id(temp pmetric.AggregationTemporality) identity.Stream { - m := s.with(s.base) - m.Sum().SetAggregationTemporality(temp) - - mid := identity.OfMetric(s.Ident().Scope(), m) - return identity.OfStream(mid, s.base) -} - -func (s SumBuilder) point(start, ts pcommon.Timestamp, value int64) data.Number { - dp := s.base.Clone() - dp.SetStartTimestamp(start) - dp.SetTimestamp(ts) - dp.SetIntValue(value) - return dp -} - -func stream() SumBuilder { - sum := random.Sum() - _, base := sum.Stream() - return SumBuilder{Metric: sum, base: base} -} - -func TestIgnore(t *testing.T) { - proc, sink := setup(t, nil) - - dir := "./testdata/notemporality-ignored" - open := func(file string) pmetric.Metrics { - t.Helper() - md, err := golden.ReadMetrics(filepath.Join(dir, file)) - require.NoError(t, err) - return md + data, err := json.Marshal(tmp) + if err != nil { + return err } - - in := open("in.yaml") - out := open("out.yaml") - - ctx := context.Background() - - err := proc.ConsumeMetrics(ctx, in) - require.NoError(t, err) - - if diff := compare.Diff([]pmetric.Metrics{out}, sink.AllMetrics()); diff != "" { - t.Fatal(diff) + md, err := (&pmetric.JSONUnmarshaler{}).UnmarshalMetrics(data) + if err != nil { + return err } + *(into.(*pmetric.Metrics)) = md + return nil } func TestTelemetry(t *testing.T) { diff --git a/processor/deltatocumulativeprocessor/testdata/limit/1.test b/processor/deltatocumulativeprocessor/testdata/limit/1.test new file mode 100644 index 000000000000..0acad04bf3c6 --- /dev/null +++ b/processor/deltatocumulativeprocessor/testdata/limit/1.test @@ -0,0 +1,47 @@ +-- in -- +resourceMetrics: + - schemaUrl: https://test.com/resource + scopeMetrics: + - schemaUrl: https://test.com/scope + scope: + name: Test + version: 1.2.3 + metrics: + - name: sum + sum: + aggregationTemporality: 1 # delta + dataPoints: + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "0"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "1"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "2"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "3"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "4"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "5"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "6"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "7"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "8"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "9"}}]} + +-- out -- +resourceMetrics: + - schemaUrl: https://test.com/resource + scopeMetrics: + - schemaUrl: https://test.com/scope + scope: + name: Test + version: 1.2.3 + metrics: + - name: sum + sum: + aggregationTemporality: 2 # cumulative + dataPoints: + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "0"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "1"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "2"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "3"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "4"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "5"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "6"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "7"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "8"}}]} + - {timeUnixNano: 1, asDouble: 1, attributes: [{key: series, value: {stringValue: "9"}}]} diff --git a/processor/deltatocumulativeprocessor/testdata/limit/2.test b/processor/deltatocumulativeprocessor/testdata/limit/2.test new file mode 100644 index 000000000000..20cd03a7db41 --- /dev/null +++ b/processor/deltatocumulativeprocessor/testdata/limit/2.test @@ -0,0 +1,49 @@ +-- in -- +resourceMetrics: + - schemaUrl: https://test.com/resource + scopeMetrics: + - schemaUrl: https://test.com/scope + scope: + name: Test + version: 1.2.3 + metrics: + - name: sum + sum: + aggregationTemporality: 1 # delta + dataPoints: + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "0"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "1"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "2"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "3"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "4"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "5"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "6"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "7"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "8"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "9"}}]} + - {timeUnixNano: 2, asDouble: 1, attributes: [{key: series, value: {stringValue: "x"}}]} # will exceed limit + +-- out -- +resourceMetrics: + - schemaUrl: https://test.com/resource + scopeMetrics: + - schemaUrl: https://test.com/scope + scope: + name: Test + version: 1.2.3 + metrics: + - name: sum + sum: + aggregationTemporality: 2 # cumulative + dataPoints: + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "0"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "1"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "2"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "3"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "4"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "5"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "6"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "7"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "8"}}]} + - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "9"}}]} + # - {timeUnixNano: 2, asDouble: 2, attributes: [{key: series, value: {stringValue: "x"}}]} # dropped diff --git a/processor/deltatocumulativeprocessor/testdata/limit/config.yaml b/processor/deltatocumulativeprocessor/testdata/limit/config.yaml new file mode 100644 index 000000000000..b8642c6cd928 --- /dev/null +++ b/processor/deltatocumulativeprocessor/testdata/limit/config.yaml @@ -0,0 +1 @@ +max_streams: 10 diff --git a/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/1.test b/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/1.test new file mode 100644 index 000000000000..c7c743bcde30 --- /dev/null +++ b/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/1.test @@ -0,0 +1,57 @@ +-- in -- +resourceMetrics: + - schemaUrl: https://test.com/resource + resource: + attributes: + - key: resattr + value: { stringValue: stringoo } + scopeMetrics: + - schemaUrl: https://test.com/scope + scope: + name: Test + version: 1.2.3 + attributes: + - key: scopeattr + value: { stringValue: string } + metrics: + - name: test.gauge + gauge: + dataPoints: + - timeUnixNano: 1 + asDouble: 1 + - name: test.summary + summary: + dataPoints: + - timeUnixNano: 1 + quantileValues: + - quantile: 0.25 + value: 25 + +-- out -- +resourceMetrics: + - schemaUrl: https://test.com/resource + resource: + attributes: + - key: resattr + value: { stringValue: stringoo } + scopeMetrics: + - schemaUrl: https://test.com/scope + scope: + name: Test + version: 1.2.3 + attributes: + - key: scopeattr + value: { stringValue: string } + metrics: + - name: test.gauge + gauge: + dataPoints: + - timeUnixNano: 1 + asDouble: 1 + - name: test.summary + summary: + dataPoints: + - timeUnixNano: 1 + quantileValues: + - quantile: 0.25 + value: 25 diff --git a/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/in.yaml b/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/in.yaml deleted file mode 100644 index 095de3947de0..000000000000 --- a/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/in.yaml +++ /dev/null @@ -1,27 +0,0 @@ -resourceMetrics: - - schemaUrl: https://test.com/resource - resource: - attributes: - - key: resattr - value: { stringValue: stringoo } - scopeMetrics: - - schemaUrl: https://test.com/scope - scope: - name: Test - version: 1.2.3 - attributes: - - key: scopeattr - value: { stringValue: string } - metrics: - - name: test.gauge - gauge: - dataPoints: - - timeUnixNano: 1 - asDouble: 1 - - name: test.summary - summary: - dataPoints: - - timeUnixNano: 1 - quantileValues: - - quantile: 0.25 - value: 25 diff --git a/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/out.yaml b/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/out.yaml deleted file mode 100644 index 095de3947de0..000000000000 --- a/processor/deltatocumulativeprocessor/testdata/notemporality-ignored/out.yaml +++ /dev/null @@ -1,27 +0,0 @@ -resourceMetrics: - - schemaUrl: https://test.com/resource - resource: - attributes: - - key: resattr - value: { stringValue: stringoo } - scopeMetrics: - - schemaUrl: https://test.com/scope - scope: - name: Test - version: 1.2.3 - attributes: - - key: scopeattr - value: { stringValue: string } - metrics: - - name: test.gauge - gauge: - dataPoints: - - timeUnixNano: 1 - asDouble: 1 - - name: test.summary - summary: - dataPoints: - - timeUnixNano: 1 - quantileValues: - - quantile: 0.25 - value: 25 diff --git a/processor/deltatocumulativeprocessor/testdata/timestamps/1.test b/processor/deltatocumulativeprocessor/testdata/timestamps/1.test new file mode 100644 index 000000000000..4f6d48c54e36 --- /dev/null +++ b/processor/deltatocumulativeprocessor/testdata/timestamps/1.test @@ -0,0 +1,36 @@ +-- in -- +resourceMetrics: + - schemaUrl: https://test.com/resource + scopeMetrics: + - schemaUrl: https://test.com/scope + scope: + name: Test + version: 1.2.3 + metrics: + - name: sum + sum: + aggregationTemporality: 1 # delta + dataPoints: + - {startTimeUnixNano: 1000, timeUnixNano: 1100, asDouble: 0} + - {startTimeUnixNano: 1100, timeUnixNano: 1200, asDouble: 0} + # - {startTimeUnixNano: 1200, timeUnixNano: 1300, asDouble: 0} + - {startTimeUnixNano: 1300, timeUnixNano: 1400, asDouble: 0} # gap (previous sample missing): accept + - {startTimeUnixNano: 1200, timeUnixNano: 1300, asDouble: 0} # out of order: drop + - {startTimeUnixNano: 500, timeUnixNano: 550, asDouble: 0} # belongs to older series: drop + +-- out -- +resourceMetrics: + - schemaUrl: https://test.com/resource + scopeMetrics: + - schemaUrl: https://test.com/scope + scope: + name: Test + version: 1.2.3 + metrics: + - name: sum + sum: + aggregationTemporality: 2 # cumulative + dataPoints: + - {startTimeUnixNano: 1000, timeUnixNano: 1100, asDouble: 0} + - {startTimeUnixNano: 1000, timeUnixNano: 1200, asDouble: 0} + - {startTimeUnixNano: 1000, timeUnixNano: 1400, asDouble: 0} diff --git a/processor/deltatocumulativeprocessor/testdata/tracking/1.test b/processor/deltatocumulativeprocessor/testdata/tracking/1.test new file mode 100644 index 000000000000..76ab437989c2 --- /dev/null +++ b/processor/deltatocumulativeprocessor/testdata/tracking/1.test @@ -0,0 +1,376 @@ +-- in -- +resourceMetrics: + - resource: + attributes: + - {key: "21f4", value: {stringValue: "42c8"}} + - {key: "7e7", value: {stringValue: "4b13"}} + scopeMetrics: + - metrics: + - description: 754d + name: 29c9 + sum: + aggregationTemporality: 1 + dataPoints: + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "1100", asInt: "124"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "1700", asInt: "22"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "2400", asInt: "27"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "3100", asInt: "115"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "4600", asInt: "47"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "5200", asInt: "34"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "6400", asInt: "98"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "6800", asInt: "36"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "7100", asInt: "48"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "8200", asInt: "118"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "8300", asInt: "62"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "8700", asInt: "118"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "9000", asInt: "51"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "9400", asInt: "49"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "10200", asInt: "97"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "10400", asInt: "21"} + unit: "7337" + - description: 611d + name: "5e04" + sum: + aggregationTemporality: 1 + dataPoints: + - {attributes: [{key: "1f1d", value: {stringValue: "762b"}}], startTimeUnixNano: "1000", timeUnixNano: "1400", asInt: "84"} + - {attributes: [{key: "1f1d", value: {stringValue: "762b"}}], startTimeUnixNano: "1000", timeUnixNano: "1800", asInt: "53"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "3400", asInt: "122"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "3600", asInt: "104"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "4100", asInt: "91"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "5300", asInt: "123"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "5600", asInt: "112"} + - {attributes: [{key: "1f1d", value: {stringValue: "762b"}}], startTimeUnixNano: "1000", timeUnixNano: "7600", asInt: "83"} + - {attributes: [{key: "1f1d", value: {stringValue: "762b"}}], startTimeUnixNano: "1000", timeUnixNano: "7700", asInt: "8"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "8500", asInt: "12"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "9100", asInt: "57"} + unit: 6db0 + scope: + attributes: + - {key: "553", value: {stringValue: "144a"}} + - {key: "5ab6", value: {stringValue: "9a8"}} + name: "7715" + version: 7bfb + - metrics: + - description: 52f4 + name: 476d + sum: + aggregationTemporality: 1 + dataPoints: + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "2100", asInt: "66"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "2900", asInt: "112"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "3000", asInt: "63"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "3900", asInt: "86"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "4000", asInt: "11"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "4200", asInt: "116"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "4300", asInt: "69"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "5400", asInt: "31"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "6500", asInt: "31"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "7300", asInt: "78"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "7900", asInt: "3"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "8000", asInt: "73"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "8900", asInt: "30"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "9500", asInt: "85"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "9600", asInt: "52"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "9900", asInt: "66"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "10000", asInt: "43"} + unit: 331a + - description: "3985" + name: "2128" + sum: + aggregationTemporality: 1 + dataPoints: + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "1500", asInt: "63"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "1600", asInt: "26"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "2500", asInt: "105"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "3200", asInt: "68"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "5500", asInt: "8"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "5800", asInt: "16"} + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "6100", asInt: "5"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "6700", asInt: "78"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "7000", asInt: "1"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "7500", asInt: "10"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "9800", asInt: "119"} + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "10300", asInt: "80"} + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "10500", asInt: "49"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "10600", asInt: "62"} + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "10700", asInt: "87"} + unit: 164b + scope: + attributes: + - {key: "6c70", value: {stringValue: "2773"}} + - {key: "766b", value: {stringValue: "370c"}} + name: 6c45 + version: 74f9 + - resource: + attributes: + - {key: "2aa", value: {stringValue: "7f34"}} + - {key: "261e", value: {stringValue: "3076"}} + scopeMetrics: + - metrics: + - description: 2d50 + name: "5863" + sum: + aggregationTemporality: 1 + dataPoints: + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "2200", asInt: "52"} + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "2700", asInt: "46"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "4400", asInt: "84"} + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "4700", asInt: "23"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "5100", asInt: "21"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "5900", asInt: "54"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "6300", asInt: "120"} + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "7800", asInt: "112"} + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "9300", asInt: "57"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "10800", asInt: "55"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "10900", asInt: "29"} + unit: 541a + - description: 6b2c + name: 430c + sum: + aggregationTemporality: 1 + dataPoints: + - {attributes: [{key: "1ac7", value: {stringValue: "5b1f"}}], startTimeUnixNano: "1000", timeUnixNano: "1200", asInt: "51"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "1900", asInt: "50"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "2800", asInt: "86"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "3300", asInt: "79"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "3500", asInt: "97"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "6200", asInt: "58"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "6900", asInt: "96"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "7200", asInt: "34"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "7400", asInt: "5"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "8800", asInt: "117"} + unit: 785e + scope: + attributes: + - {key: "509", value: {stringValue: "2bcd"}} + - {key: "2aec", value: {stringValue: "390b"}} + name: "6812" + version: 17ce + - metrics: + - description: c6a + name: 3ccc + sum: + aggregationTemporality: 1 + dataPoints: + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "1300", asInt: "77"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "2300", asInt: "8"} + - {attributes: [{key: "306c", value: {stringValue: "3c61"}}], startTimeUnixNano: "1000", timeUnixNano: "3800", asInt: "41"} + - {attributes: [{key: "306c", value: {stringValue: "3c61"}}], startTimeUnixNano: "1000", timeUnixNano: "5700", asInt: "31"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "6600", asInt: "38"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "8400", asInt: "13"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "8600", asInt: "106"} + - {attributes: [{key: "306c", value: {stringValue: "3c61"}}], startTimeUnixNano: "1000", timeUnixNano: "9200", asInt: "49"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "9700", asInt: "0"} + unit: 1adc + - description: 20fb + name: 61b6 + sum: + aggregationTemporality: 1 + dataPoints: + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "1000", asInt: "45"} + - {attributes: [{key: "2afe", value: {stringValue: "22f1"}}], startTimeUnixNano: "1000", timeUnixNano: "2000", asInt: "53"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "2600", asInt: "47"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "3700", asInt: "61"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "4500", asInt: "27"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "4800", asInt: "37"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "4900", asInt: "100"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "5000", asInt: "73"} + - {attributes: [{key: "2afe", value: {stringValue: "22f1"}}], startTimeUnixNano: "1000", timeUnixNano: "6000", asInt: "71"} + - {attributes: [{key: "2afe", value: {stringValue: "22f1"}}], startTimeUnixNano: "1000", timeUnixNano: "8100", asInt: "55"} + - {attributes: [{key: "2afe", value: {stringValue: "22f1"}}], startTimeUnixNano: "1000", timeUnixNano: "10100", asInt: "71"} + unit: "5679" + scope: + attributes: + - {key: "14cf", value: {stringValue: "64ea"}} + - {key: "67ef", value: {stringValue: "4299"}} + name: 58a7 + version: 1cd0 +-- out -- +resourceMetrics: + - resource: + attributes: + - {key: "21f4", value: {stringValue: "42c8"}} + - {key: "7e7", value: {stringValue: "4b13"}} + scopeMetrics: + - metrics: + - description: 754d + name: 29c9 + sum: + aggregationTemporality: 2 + dataPoints: + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "1100", asInt: "124"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "1700", asInt: "22"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "2400", asInt: "49"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "3100", asInt: "239"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "4600", asInt: "286"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "5200", asInt: "320"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "6400", asInt: "418"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "6800", asInt: "85"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "7100", asInt: "133"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "8200", asInt: "251"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "8300", asInt: "313"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "8700", asInt: "431"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "9000", asInt: "469"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "9400", asInt: "518"} + - {attributes: [{key: "10d", value: {stringValue: "68c6"}}], startTimeUnixNano: "1000", timeUnixNano: "10200", asInt: "615"} + - {attributes: [{key: "7861", value: {stringValue: "3d13"}}], startTimeUnixNano: "1000", timeUnixNano: "10400", asInt: "452"} + unit: "7337" + - description: 611d + name: "5e04" + sum: + aggregationTemporality: 2 + dataPoints: + - {attributes: [{key: "1f1d", value: {stringValue: "762b"}}], startTimeUnixNano: "1000", timeUnixNano: "1400", asInt: "84"} + - {attributes: [{key: "1f1d", value: {stringValue: "762b"}}], startTimeUnixNano: "1000", timeUnixNano: "1800", asInt: "137"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "3400", asInt: "122"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "3600", asInt: "226"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "4100", asInt: "317"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "5300", asInt: "440"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "5600", asInt: "552"} + - {attributes: [{key: "1f1d", value: {stringValue: "762b"}}], startTimeUnixNano: "1000", timeUnixNano: "7600", asInt: "220"} + - {attributes: [{key: "1f1d", value: {stringValue: "762b"}}], startTimeUnixNano: "1000", timeUnixNano: "7700", asInt: "228"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "8500", asInt: "564"} + - {attributes: [{key: "6eeb", value: {stringValue: "e44"}}], startTimeUnixNano: "1000", timeUnixNano: "9100", asInt: "621"} + unit: 6db0 + scope: + attributes: + - {key: "553", value: {stringValue: "144a"}} + - {key: "5ab6", value: {stringValue: "9a8"}} + name: "7715" + version: 7bfb + - metrics: + - description: 52f4 + name: 476d + sum: + aggregationTemporality: 2 + dataPoints: + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "2100", asInt: "66"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "2900", asInt: "112"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "3000", asInt: "129"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "3900", asInt: "198"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "4000", asInt: "209"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "4200", asInt: "325"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "4300", asInt: "394"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "5400", asInt: "425"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "6500", asInt: "456"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "7300", asInt: "207"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "7900", asInt: "210"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "8000", asInt: "529"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "8900", asInt: "559"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "9500", asInt: "644"} + - {attributes: [{key: "18d7", value: {stringValue: "6b33"}}], startTimeUnixNano: "1000", timeUnixNano: "9600", asInt: "696"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "9900", asInt: "276"} + - {attributes: [{key: "19b0", value: {stringValue: "114f"}}], startTimeUnixNano: "1000", timeUnixNano: "10000", asInt: "319"} + unit: 331a + - description: "3985" + name: "2128" + sum: + aggregationTemporality: 2 + dataPoints: + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "1500", asInt: "63"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "1600", asInt: "26"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "2500", asInt: "131"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "3200", asInt: "199"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "5500", asInt: "207"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "5800", asInt: "223"} + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "6100", asInt: "68"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "6700", asInt: "301"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "7000", asInt: "302"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "7500", asInt: "312"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "9800", asInt: "431"} + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "10300", asInt: "148"} + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "10500", asInt: "197"} + - {attributes: [{key: "6c73", value: {stringValue: "2fc5"}}], startTimeUnixNano: "1000", timeUnixNano: "10600", asInt: "493"} + - {attributes: [{key: "b0", value: {stringValue: "3b97"}}], startTimeUnixNano: "1000", timeUnixNano: "10700", asInt: "284"} + unit: 164b + scope: + attributes: + - {key: "6c70", value: {stringValue: "2773"}} + - {key: "766b", value: {stringValue: "370c"}} + name: 6c45 + version: 74f9 + - resource: + attributes: + - {key: "2aa", value: {stringValue: "7f34"}} + - {key: "261e", value: {stringValue: "3076"}} + scopeMetrics: + - metrics: + - description: 2d50 + name: "5863" + sum: + aggregationTemporality: 2 + dataPoints: + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "2200", asInt: "52"} + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "2700", asInt: "98"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "4400", asInt: "84"} + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "4700", asInt: "121"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "5100", asInt: "105"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "5900", asInt: "159"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "6300", asInt: "279"} + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "7800", asInt: "233"} + - {attributes: [{key: "27bd", value: {stringValue: "21a"}}], startTimeUnixNano: "1000", timeUnixNano: "9300", asInt: "290"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "10800", asInt: "334"} + - {attributes: [{key: "33cf", value: {stringValue: "23b0"}}], startTimeUnixNano: "1000", timeUnixNano: "10900", asInt: "363"} + unit: 541a + - description: 6b2c + name: 430c + sum: + aggregationTemporality: 2 + dataPoints: + - {attributes: [{key: "1ac7", value: {stringValue: "5b1f"}}], startTimeUnixNano: "1000", timeUnixNano: "1200", asInt: "51"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "1900", asInt: "50"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "2800", asInt: "136"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "3300", asInt: "215"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "3500", asInt: "312"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "6200", asInt: "370"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "6900", asInt: "466"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "7200", asInt: "500"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "7400", asInt: "505"} + - {attributes: [{key: "50bf", value: {stringValue: "4e9a"}}], startTimeUnixNano: "1000", timeUnixNano: "8800", asInt: "622"} + unit: 785e + scope: + attributes: + - {key: "509", value: {stringValue: "2bcd"}} + - {key: "2aec", value: {stringValue: "390b"}} + name: "6812" + version: 17ce + - metrics: + - description: c6a + name: 3ccc + sum: + aggregationTemporality: 2 + dataPoints: + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "1300", asInt: "77"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "2300", asInt: "85"} + - {attributes: [{key: "306c", value: {stringValue: "3c61"}}], startTimeUnixNano: "1000", timeUnixNano: "3800", asInt: "41"} + - {attributes: [{key: "306c", value: {stringValue: "3c61"}}], startTimeUnixNano: "1000", timeUnixNano: "5700", asInt: "72"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "6600", asInt: "123"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "8400", asInt: "136"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "8600", asInt: "242"} + - {attributes: [{key: "306c", value: {stringValue: "3c61"}}], startTimeUnixNano: "1000", timeUnixNano: "9200", asInt: "121"} + - {attributes: [{key: "75d9", value: {stringValue: "4b59"}}], startTimeUnixNano: "1000", timeUnixNano: "9700", asInt: "242"} + unit: 1adc + - description: 20fb + name: 61b6 + sum: + aggregationTemporality: 2 + dataPoints: + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "1000", asInt: "45"} + - {attributes: [{key: "2afe", value: {stringValue: "22f1"}}], startTimeUnixNano: "1000", timeUnixNano: "2000", asInt: "53"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "2600", asInt: "92"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "3700", asInt: "153"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "4500", asInt: "180"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "4800", asInt: "217"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "4900", asInt: "317"} + - {attributes: [{key: "2fad", value: {stringValue: "3ea1"}}], startTimeUnixNano: "1000", timeUnixNano: "5000", asInt: "390"} + - {attributes: [{key: "2afe", value: {stringValue: "22f1"}}], startTimeUnixNano: "1000", timeUnixNano: "6000", asInt: "124"} + - {attributes: [{key: "2afe", value: {stringValue: "22f1"}}], startTimeUnixNano: "1000", timeUnixNano: "8100", asInt: "179"} + - {attributes: [{key: "2afe", value: {stringValue: "22f1"}}], startTimeUnixNano: "1000", timeUnixNano: "10100", asInt: "250"} + unit: "5679" + scope: + attributes: + - {key: "14cf", value: {stringValue: "64ea"}} + - {key: "67ef", value: {stringValue: "4299"}} + name: 58a7 + version: 1cd0 From 29d0174a544b655647c47ffcc70cbb505c8becf9 Mon Sep 17 00:00:00 2001 From: Alex Boten <223565+codeboten@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:18:14 -0700 Subject: [PATCH 15/21] [chore] update test to use Contains (#35875) This follows the recommendations from the new lint version Signed-off-by: Alex Boten <223565+codeboten@users.noreply.github.com> --- receiver/snmpreceiver/client_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/receiver/snmpreceiver/client_test.go b/receiver/snmpreceiver/client_test.go index 8cfb26ad1f09..5bc38d09ad29 100644 --- a/receiver/snmpreceiver/client_test.go +++ b/receiver/snmpreceiver/client_test.go @@ -8,7 +8,6 @@ import ( "fmt" "math" "strconv" - "strings" "testing" "github.com/gosnmp/gosnmp" @@ -83,9 +82,9 @@ func TestNewClient(t *testing.T) { func compareConfigToClient(t *testing.T, client *snmpClient, cfg *Config) { t.Helper() - require.True(t, strings.Contains(cfg.Endpoint, client.client.GetTarget())) - require.True(t, strings.Contains(cfg.Endpoint, strconv.FormatInt(int64(client.client.GetPort()), 10))) - require.True(t, strings.Contains(cfg.Endpoint, client.client.GetTransport())) + require.Contains(t, cfg.Endpoint, client.client.GetTarget()) + require.Contains(t, cfg.Endpoint, strconv.FormatInt(int64(client.client.GetPort()), 10)) + require.Contains(t, cfg.Endpoint, client.client.GetTransport()) switch cfg.Version { case "v1": require.Equal(t, gosnmp.Version1, client.client.GetVersion()) From 17225bbb96a877c5c40b394273792cca6ba5b45d Mon Sep 17 00:00:00 2001 From: Mike Goldsmith Date: Fri, 18 Oct 2024 19:41:10 +0100 Subject: [PATCH 16/21] [chore] Fix typos in Azure Receiver readme (#35871) #### Description Fixes some typos in the Azure Receiver readme. --- receiver/azureeventhubreceiver/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/receiver/azureeventhubreceiver/README.md b/receiver/azureeventhubreceiver/README.md index bf874629c6ed..48404410a48c 100644 --- a/receiver/azureeventhubreceiver/README.md +++ b/receiver/azureeventhubreceiver/README.md @@ -45,8 +45,8 @@ Default: "azure" ### apply_semantic_conventions (optional) Determines whether Azure Resource Logs are translated into OpenTelemetry Logs using semantic -convention attrobute names or not. When not applying semantic conventions, the log entry -attribute nates are copied without any changes. +convention attribute names or not. When not applying semantic conventions, the log entry +attribute names are copied without any changes. Default: `false` (semantic conventions are not applied) From aff4d9a81c6a4225b1203ba6391a7e2ce72ce901 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Sat, 19 Oct 2024 02:41:25 +0800 Subject: [PATCH 17/21] [chore] fix update to golang-lru/v2 (#35847) #### Description Fixes incomplete mechanical update in https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/30890. The new version of golang-lru uses type params. Recreates https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/35824, which Renovate kindly closed without merging :smiling_face_with_tear: #### Link to tracking issue Supersedes https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/30890 #### Testing Ran the unit tests. #### Documentation N/A Co-authored-by: Alex Boten <223565+codeboten@users.noreply.github.com> --- connector/spanmetricsconnector/go.mod | 1 - connector/spanmetricsconnector/go.sum | 2 -- .../spanmetricsconnector/internal/cache/cache.go | 12 ++++++------ extension/observer/ecsobserver/fetcher.go | 14 +++++++------- extension/observer/ecsobserver/go.mod | 2 +- extension/observer/ecsobserver/go.sum | 4 ++-- 6 files changed, 16 insertions(+), 19 deletions(-) diff --git a/connector/spanmetricsconnector/go.mod b/connector/spanmetricsconnector/go.mod index 0b7a013b1136..a42b3ba0bf0d 100644 --- a/connector/spanmetricsconnector/go.mod +++ b/connector/spanmetricsconnector/go.mod @@ -3,7 +3,6 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanm go 1.22.0 require ( - github.com/hashicorp/golang-lru v1.0.2 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jonboulle/clockwork v0.4.0 github.com/lightstep/go-expohisto v1.0.0 diff --git a/connector/spanmetricsconnector/go.sum b/connector/spanmetricsconnector/go.sum index b138d0515666..3900a7be0797 100644 --- a/connector/spanmetricsconnector/go.sum +++ b/connector/spanmetricsconnector/go.sum @@ -19,8 +19,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= -github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= diff --git a/connector/spanmetricsconnector/internal/cache/cache.go b/connector/spanmetricsconnector/internal/cache/cache.go index 2559d42eadf7..c59eadd3ebcc 100644 --- a/connector/spanmetricsconnector/internal/cache/cache.go +++ b/connector/spanmetricsconnector/internal/cache/cache.go @@ -4,7 +4,7 @@ package cache // import "github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector/internal/cache" import ( - "github.com/hashicorp/golang-lru/simplelru" + "github.com/hashicorp/golang-lru/v2/simplelru" ) // Cache consists of an LRU cache and the evicted items from the LRU cache. @@ -15,15 +15,15 @@ import ( // // Important: This implementation is non-thread safe. type Cache[K comparable, V any] struct { - lru simplelru.LRUCache + lru *simplelru.LRU[K, V] evictedItems map[K]V } // NewCache creates a Cache. func NewCache[K comparable, V any](size int) (*Cache[K, V], error) { evictedItems := make(map[K]V) - lruCache, err := simplelru.NewLRU(size, func(key any, value any) { - evictedItems[key.(K)] = value.(V) + lruCache, err := simplelru.NewLRU(size, func(key K, value V) { + evictedItems[key] = value }) if err != nil { return nil, err @@ -51,7 +51,7 @@ func (c *Cache[K, V]) Add(key K, value V) bool { // Get an item from the LRU cache or evicted items. func (c *Cache[K, V]) Get(key K) (V, bool) { if val, ok := c.lru.Get(key); ok { - return val.(V), ok + return val, ok } val, ok := c.evictedItems[key] @@ -85,7 +85,7 @@ func (c *Cache[K, V]) ForEach(fn func(k K, v V)) { for _, k := range c.lru.Keys() { v, ok := c.lru.Get(k) if ok { - fn(k.(K), v.(V)) + fn(k, v) } } diff --git a/extension/observer/ecsobserver/fetcher.go b/extension/observer/ecsobserver/fetcher.go index cf21efafcb4d..4276ddfacf6f 100644 --- a/extension/observer/ecsobserver/fetcher.go +++ b/extension/observer/ecsobserver/fetcher.go @@ -14,7 +14,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ecs" - "github.com/hashicorp/golang-lru/simplelru" + "github.com/hashicorp/golang-lru/v2/simplelru" "go.uber.org/zap" ) @@ -50,8 +50,8 @@ type taskFetcher struct { ecs ecsClient ec2 ec2Client cluster string - taskDefCache simplelru.LRUCache - ec2Cache simplelru.LRUCache + taskDefCache *simplelru.LRU[string, *ecs.TaskDefinition] + ec2Cache *simplelru.LRU[string, *ec2.Instance] serviceNameFilter serviceNameFilter } @@ -81,11 +81,11 @@ func newTaskFetcherFromConfig(cfg Config, logger *zap.Logger) (*taskFetcher, err func newTaskFetcher(opts taskFetcherOptions) (*taskFetcher, error) { // Init cache - taskDefCache, err := simplelru.NewLRU(taskDefCacheSize, nil) + taskDefCache, err := simplelru.NewLRU[string, *ecs.TaskDefinition](taskDefCacheSize, nil) if err != nil { return nil, err } - ec2Cache, err := simplelru.NewLRU(ec2CacheSize, nil) + ec2Cache, err := simplelru.NewLRU[string, *ec2.Instance](ec2CacheSize, nil) if err != nil { return nil, err } @@ -205,7 +205,7 @@ func (f *taskFetcher) attachTaskDefinition(ctx context.Context, tasks []*ecs.Tas } var def *ecs.TaskDefinition if cached, ok := f.taskDefCache.Get(arn); ok { - def = cached.(*ecs.TaskDefinition) + def = cached } else { res, err := svc.DescribeTaskDefinitionWithContext(ctx, &ecs.DescribeTaskDefinitionInput{ TaskDefinition: aws.String(arn), @@ -251,7 +251,7 @@ func (f *taskFetcher) attachContainerInstance(ctx context.Context, tasks []*task for instanceArn := range ciToEC2 { cached, ok := f.ec2Cache.Get(instanceArn) if ok { - ciToEC2[instanceArn] = cached.(*ec2.Instance) // use value from cache + ciToEC2[instanceArn] = cached // use value from cache } else { instanceList = append(instanceList, aws.String(instanceArn)) } diff --git a/extension/observer/ecsobserver/go.mod b/extension/observer/ecsobserver/go.mod index c5539879449f..3866d9b43261 100644 --- a/extension/observer/ecsobserver/go.mod +++ b/extension/observer/ecsobserver/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/aws/aws-sdk-go v1.55.5 - github.com/hashicorp/golang-lru v1.0.2 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.111.1-0.20241008154146-ea48c09c31ae go.opentelemetry.io/collector/component/componentstatus v0.111.1-0.20241008154146-ea48c09c31ae diff --git a/extension/observer/ecsobserver/go.sum b/extension/observer/ecsobserver/go.sum index 08c9eae60fb9..c5c61a6b49a6 100644 --- a/extension/observer/ecsobserver/go.sum +++ b/extension/observer/ecsobserver/go.sum @@ -16,8 +16,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= -github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= From 33a54575731944cba22229d14098c45bb5b6c285 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:29:33 -0700 Subject: [PATCH 18/21] fix(deps): update module github.com/antonboom/testifylint to v1.5.0 (#35797) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [github.com/Antonboom/testifylint](https://redirect.github.com/Antonboom/testifylint) | `v1.4.3` -> `v1.5.0` | [![age](https://developer.mend.io/api/mc/badges/age/go/github.com%2fAntonboom%2ftestifylint/v1.5.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://developer.mend.io/api/mc/badges/adoption/go/github.com%2fAntonboom%2ftestifylint/v1.5.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://developer.mend.io/api/mc/badges/compatibility/go/github.com%2fAntonboom%2ftestifylint/v1.4.3/v1.5.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://developer.mend.io/api/mc/badges/confidence/go/github.com%2fAntonboom%2ftestifylint/v1.4.3/v1.5.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | --- > [!WARNING] > Some dependencies could not be looked up. Check the Dependency Dashboard for more information. --- ### Release Notes
Antonboom/testifylint (github.com/Antonboom/testifylint) ### [`v1.5.0`](https://redirect.github.com/Antonboom/testifylint/releases/tag/v1.5.0): New `contains`, `regexp`, `encoded-compare'`! [Compare Source](https://redirect.github.com/Antonboom/testifylint/compare/v1.4.3...v1.5.0) #### What's Changed - ci: checkout before setup-go by [@​mmorel-35](https://redirect.github.com/mmorel-35) in [https://github.com/Antonboom/testifylint/pull/137](https://redirect.github.com/Antonboom/testifylint/pull/137) - new checker 'regexp' by [@​Antonboom](https://redirect.github.com/Antonboom) in [https://github.com/Antonboom/testifylint/pull/136](https://redirect.github.com/Antonboom/testifylint/pull/136) - new checker `contains` by [@​mmorel-35](https://redirect.github.com/mmorel-35) in [https://github.com/Antonboom/testifylint/pull/152](https://redirect.github.com/Antonboom/testifylint/pull/152) - build(deps): bump golang.org/x/tools from 0.21.0 to 0.22.0 by [@​dependabot](https://redirect.github.com/dependabot) in [https://github.com/Antonboom/testifylint/pull/163](https://redirect.github.com/Antonboom/testifylint/pull/163) - build(deps): bump golang.org/x/tools from 0.22.0 to 0.23.0 by [@​dependabot](https://redirect.github.com/dependabot) in [https://github.com/Antonboom/testifylint/pull/168](https://redirect.github.com/Antonboom/testifylint/pull/168) - enable several rules from revive by [@​mmorel-35](https://redirect.github.com/mmorel-35) in [https://github.com/Antonboom/testifylint/pull/164](https://redirect.github.com/Antonboom/testifylint/pull/164) - build(deps): bump golang.org/x/tools from 0.23.0 to 0.24.0 by [@​dependabot](https://redirect.github.com/dependabot) in [https://github.com/Antonboom/testifylint/pull/173](https://redirect.github.com/Antonboom/testifylint/pull/173) - useless-assert: more cases by [@​Antonboom](https://redirect.github.com/Antonboom) in [https://github.com/Antonboom/testifylint/pull/182](https://redirect.github.com/Antonboom/testifylint/pull/182) - formatter: support require only and suite only packages by [@​Antonboom](https://redirect.github.com/Antonboom) in [https://github.com/Antonboom/testifylint/pull/188](https://redirect.github.com/Antonboom/testifylint/pull/188) - new checker 'encoded-compare' by [@​Antonboom](https://redirect.github.com/Antonboom) in [https://github.com/Antonboom/testifylint/pull/179](https://redirect.github.com/Antonboom/testifylint/pull/179) - empty: adjust report for NotEmpty+len cases by [@​Antonboom](https://redirect.github.com/Antonboom) in [https://github.com/Antonboom/testifylint/pull/189](https://redirect.github.com/Antonboom/testifylint/pull/189) - error-nil: support Empty and Zero error by [@​mmorel-35](https://redirect.github.com/mmorel-35) in [https://github.com/Antonboom/testifylint/pull/153](https://redirect.github.com/Antonboom/testifylint/pull/153) - Go 1.23 by [@​Antonboom](https://redirect.github.com/Antonboom) in [https://github.com/Antonboom/testifylint/pull/191](https://redirect.github.com/Antonboom/testifylint/pull/191) **Full Changelog**: https://github.com/Antonboom/testifylint/compare/v1.4.3...v1.5.0
--- ### Configuration 📅 **Schedule**: Branch creation - "on tuesday" (UTC), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/open-telemetry/opentelemetry-collector-contrib). --------- Signed-off-by: Alex Boten <223565+codeboten@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: opentelemetrybot <107717825+opentelemetrybot@users.noreply.github.com> Co-authored-by: Alex Boten <223565+codeboten@users.noreply.github.com> --- .../awsemfexporter/grouped_metric_test.go | 2 +- .../internal/translator/cause_test.go | 4 +- .../internal/translator/http_test.go | 31 ++++---- .../internal/translator/segment_test.go | 67 +++++++++-------- .../internal/translator/service_test.go | 5 +- .../internal/translator/span_links_test.go | 71 +++++++++---------- .../internal/translator/sql_test.go | 3 +- .../elasticsearchexporter/exporter_test.go | 20 +++--- .../logs_exporter_test.go | 3 +- exporter/mezmoexporter/exporter_test.go | 2 +- .../pulsarexporter/jaeger_marshaler_test.go | 2 +- exporter/pulsarexporter/marshaler_test.go | 4 +- .../metricdata_to_splunk_test.go | 2 +- .../syslogexporter/rfc5424_formatter_test.go | 9 ++- .../extension_test.go | 2 +- .../jsonlogencodingextension/json_test.go | 4 +- .../healthcheckextension/integration_test.go | 2 +- .../healthcheckv2extension/extension_test.go | 2 +- .../internal/http/server_test.go | 2 +- .../httpforwarderextension/extension_test.go | 2 +- .../httpforwarderextension/factory_test.go | 3 +- .../internal/http_test.go | 2 +- internal/tools/go.mod | 6 +- internal/tools/go.sum | 4 +- pkg/stanza/operator/operatortest/confmap.go | 3 +- processor/remotetapprocessor/server_test.go | 6 +- .../internal/cache/lru_cache_test.go | 4 +- ..._exponential_hist_to_explicit_hist_test.go | 3 +- .../handlecount/handles_windows_test.go | 3 +- receiver/splunkhecreceiver/receiver_test.go | 16 ++--- 30 files changed, 140 insertions(+), 149 deletions(-) diff --git a/exporter/awsemfexporter/grouped_metric_test.go b/exporter/awsemfexporter/grouped_metric_test.go index e7c176f82b01..d2b1a86610f0 100644 --- a/exporter/awsemfexporter/grouped_metric_test.go +++ b/exporter/awsemfexporter/grouped_metric_test.go @@ -435,7 +435,7 @@ func TestAddKubernetesWrapper(t *testing.T) { jsonBytes, _ := json.Marshal(expectedCreatedObj) addKubernetesWrapper(inputs) - assert.Equal(t, string(jsonBytes), inputs["kubernetes"], "The created and expected objects should be the same") + assert.JSONEq(t, string(jsonBytes), inputs["kubernetes"], "The created and expected objects should be the same") }) } diff --git a/exporter/awsxrayexporter/internal/translator/cause_test.go b/exporter/awsxrayexporter/internal/translator/cause_test.go index 0369d7066d73..6f4dbfcd9eb0 100644 --- a/exporter/awsxrayexporter/internal/translator/cause_test.go +++ b/exporter/awsxrayexporter/internal/translator/cause_test.go @@ -193,7 +193,7 @@ func TestCauseWithStatusMessage(t *testing.T) { require.NoError(t, w.Encode(cause)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, errorMsg)) + assert.Contains(t, jsonStr, errorMsg) } func TestCauseWithHttpStatusMessage(t *testing.T) { @@ -218,7 +218,7 @@ func TestCauseWithHttpStatusMessage(t *testing.T) { require.NoError(t, w.Encode(cause)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, errorMsg)) + assert.Contains(t, jsonStr, errorMsg) } func TestCauseWithZeroStatusMessageAndFaultHttpCode(t *testing.T) { diff --git a/exporter/awsxrayexporter/internal/translator/http_test.go b/exporter/awsxrayexporter/internal/translator/http_test.go index f34c4e82af59..d0f8b375c72f 100644 --- a/exporter/awsxrayexporter/internal/translator/http_test.go +++ b/exporter/awsxrayexporter/internal/translator/http_test.go @@ -4,7 +4,6 @@ package translator import ( - "strings" "testing" "time" @@ -31,7 +30,7 @@ func TestClientSpanWithURLAttribute(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestClientSpanWithURLAttributeStable(t *testing.T) { @@ -49,7 +48,7 @@ func TestClientSpanWithURLAttributeStable(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestClientSpanWithSchemeHostTargetAttributes(t *testing.T) { @@ -70,7 +69,7 @@ func TestClientSpanWithSchemeHostTargetAttributes(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestClientSpanWithPeerAttributes(t *testing.T) { @@ -95,7 +94,7 @@ func TestClientSpanWithPeerAttributes(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "http://kb234.example.com:8080/users/junit")) + assert.Contains(t, jsonStr, "http://kb234.example.com:8080/users/junit") } func TestClientSpanWithPeerAttributesStable(t *testing.T) { @@ -120,7 +119,7 @@ func TestClientSpanWithPeerAttributesStable(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "http://kb234.example.com:8080/users/junit")) + assert.Contains(t, jsonStr, "http://kb234.example.com:8080/users/junit") } func TestClientSpanWithHttpPeerAttributes(t *testing.T) { @@ -168,7 +167,7 @@ func TestClientSpanWithPeerIp4Attributes(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "http://10.8.17.36:8080/users/junit")) + assert.Contains(t, jsonStr, "http://10.8.17.36:8080/users/junit") } func TestClientSpanWithPeerIp6Attributes(t *testing.T) { @@ -187,7 +186,7 @@ func TestClientSpanWithPeerIp6Attributes(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://2001:db8:85a3::8a2e:370:7334/users/junit")) + assert.Contains(t, jsonStr, "https://2001:db8:85a3::8a2e:370:7334/users/junit") } func TestServerSpanWithURLAttribute(t *testing.T) { @@ -207,7 +206,7 @@ func TestServerSpanWithURLAttribute(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestServerSpanWithURLAttributeStable(t *testing.T) { @@ -227,7 +226,7 @@ func TestServerSpanWithURLAttributeStable(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestServerSpanWithSchemeHostTargetAttributes(t *testing.T) { @@ -248,7 +247,7 @@ func TestServerSpanWithSchemeHostTargetAttributes(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestServerSpanWithSchemeHostTargetAttributesStable(t *testing.T) { @@ -269,7 +268,7 @@ func TestServerSpanWithSchemeHostTargetAttributesStable(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestServerSpanWithSchemeServernamePortTargetAttributes(t *testing.T) { @@ -291,7 +290,7 @@ func TestServerSpanWithSchemeServernamePortTargetAttributes(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestServerSpanWithSchemeServernamePortTargetAttributesStable(t *testing.T) { @@ -313,7 +312,7 @@ func TestServerSpanWithSchemeServernamePortTargetAttributesStable(t *testing.T) require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "https://api.example.com/users/junit")) + assert.Contains(t, jsonStr, "https://api.example.com/users/junit") } func TestServerSpanWithSchemeNamePortTargetAttributes(t *testing.T) { @@ -337,7 +336,7 @@ func TestServerSpanWithSchemeNamePortTargetAttributes(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "http://kb234.example.com:8080/users/junit")) + assert.Contains(t, jsonStr, "http://kb234.example.com:8080/users/junit") } func TestServerSpanWithSchemeNamePortTargetAttributesStable(t *testing.T) { @@ -361,7 +360,7 @@ func TestServerSpanWithSchemeNamePortTargetAttributesStable(t *testing.T) { require.NoError(t, w.Encode(httpData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "http://kb234.example.com:8080/users/junit")) + assert.Contains(t, jsonStr, "http://kb234.example.com:8080/users/junit") } func TestSpanWithNotEnoughHTTPRequestURLAttributes(t *testing.T) { diff --git a/exporter/awsxrayexporter/internal/translator/segment_test.go b/exporter/awsxrayexporter/internal/translator/segment_test.go index 57ab63ecc726..6351ad99922c 100644 --- a/exporter/awsxrayexporter/internal/translator/segment_test.go +++ b/exporter/awsxrayexporter/internal/translator/segment_test.go @@ -7,7 +7,6 @@ import ( "crypto/rand" "encoding/binary" "fmt" - "strings" "testing" "time" @@ -61,10 +60,10 @@ func TestClientSpanWithRpcAwsSdkClientAttributes(t *testing.T) { assert.NotNil(t, jsonStr) assert.NoError(t, err) - assert.True(t, strings.Contains(jsonStr, "DynamoDB")) - assert.True(t, strings.Contains(jsonStr, "GetItem")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "DynamoDB") + assert.Contains(t, jsonStr, "GetItem") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func TestClientSpanWithLegacyAwsSdkClientAttributes(t *testing.T) { @@ -94,10 +93,10 @@ func TestClientSpanWithLegacyAwsSdkClientAttributes(t *testing.T) { assert.NotNil(t, jsonStr) assert.NoError(t, err) - assert.True(t, strings.Contains(jsonStr, "DynamoDB")) - assert.True(t, strings.Contains(jsonStr, "GetItem")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "DynamoDB") + assert.Contains(t, jsonStr, "GetItem") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func TestClientSpanWithPeerService(t *testing.T) { @@ -253,8 +252,8 @@ func TestClientSpanWithDbComponent(t *testing.T) { require.NoError(t, w.Encode(segment)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, spanName)) - assert.True(t, strings.Contains(jsonStr, enterpriseAppID)) + assert.Contains(t, jsonStr, spanName) + assert.Contains(t, jsonStr, enterpriseAppID) } func TestClientSpanWithHttpHost(t *testing.T) { @@ -376,9 +375,9 @@ func TestSpanWithInvalidTraceIdWithoutTimestampValidation(t *testing.T) { require.NoError(t, err) assert.NotNil(t, jsonStr) - assert.True(t, strings.Contains(jsonStr, "ProducerService")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "ProducerService") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func TestSpanWithExpiredTraceIdWithoutTimestampValidation(t *testing.T) { @@ -1088,9 +1087,9 @@ func TestClientSpanWithAwsRemoteServiceName(t *testing.T) { assert.NotNil(t, jsonStr) assert.NoError(t, err) - assert.True(t, strings.Contains(jsonStr, "PaymentService")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "PaymentService") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func TestAwsSdkSpanWithDeprecatedAwsRemoteServiceName(t *testing.T) { @@ -1115,10 +1114,10 @@ func TestAwsSdkSpanWithDeprecatedAwsRemoteServiceName(t *testing.T) { assert.NotNil(t, jsonStr) assert.NoError(t, err) - assert.True(t, strings.Contains(jsonStr, "DynamoDb")) - assert.False(t, strings.Contains(jsonStr, "DynamoDb.PutItem")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "DynamoDb") + assert.NotContains(t, jsonStr, "DynamoDb.PutItem") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func TestAwsSdkSpanWithAwsRemoteServiceName(t *testing.T) { @@ -1143,10 +1142,10 @@ func TestAwsSdkSpanWithAwsRemoteServiceName(t *testing.T) { assert.NotNil(t, jsonStr) assert.NoError(t, err) - assert.True(t, strings.Contains(jsonStr, "DynamoDb")) - assert.False(t, strings.Contains(jsonStr, "DynamoDb.PutItem")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "DynamoDb") + assert.NotContains(t, jsonStr, "DynamoDb.PutItem") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func TestProducerSpanWithAwsRemoteServiceName(t *testing.T) { @@ -1172,9 +1171,9 @@ func TestProducerSpanWithAwsRemoteServiceName(t *testing.T) { assert.NotNil(t, jsonStr) assert.NoError(t, err) - assert.True(t, strings.Contains(jsonStr, "ProducerService")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "ProducerService") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func TestConsumerSpanWithAwsRemoteServiceName(t *testing.T) { @@ -1191,9 +1190,9 @@ func TestConsumerSpanWithAwsRemoteServiceName(t *testing.T) { assert.NotNil(t, jsonStr) assert.NoError(t, err) - assert.True(t, strings.Contains(jsonStr, "ConsumerService")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "ConsumerService") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func TestServerSpanWithAwsLocalServiceName(t *testing.T) { @@ -1219,9 +1218,9 @@ func TestServerSpanWithAwsLocalServiceName(t *testing.T) { assert.NotNil(t, jsonStr) assert.NoError(t, err) - assert.True(t, strings.Contains(jsonStr, "PaymentLocalService")) - assert.False(t, strings.Contains(jsonStr, user)) - assert.False(t, strings.Contains(jsonStr, "user")) + assert.Contains(t, jsonStr, "PaymentLocalService") + assert.NotContains(t, jsonStr, user) + assert.NotContains(t, jsonStr, "user") } func validateLocalRootDependencySubsegment(t *testing.T, segment *awsxray.Segment, span ptrace.Span, parentID string) { diff --git a/exporter/awsxrayexporter/internal/translator/service_test.go b/exporter/awsxrayexporter/internal/translator/service_test.go index 9d26c5f2f098..81745b031aba 100644 --- a/exporter/awsxrayexporter/internal/translator/service_test.go +++ b/exporter/awsxrayexporter/internal/translator/service_test.go @@ -4,7 +4,6 @@ package translator import ( - "strings" "testing" "github.com/stretchr/testify/assert" @@ -23,7 +22,7 @@ func TestServiceFromResource(t *testing.T) { require.NoError(t, w.Encode(service)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "semver:1.1.4")) + assert.Contains(t, jsonStr, "semver:1.1.4") } func TestServiceFromResourceWithNoServiceVersion(t *testing.T) { @@ -36,7 +35,7 @@ func TestServiceFromResourceWithNoServiceVersion(t *testing.T) { require.NoError(t, w.Encode(service)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "v1")) + assert.Contains(t, jsonStr, "v1") } func TestServiceFromNullResource(t *testing.T) { diff --git a/exporter/awsxrayexporter/internal/translator/span_links_test.go b/exporter/awsxrayexporter/internal/translator/span_links_test.go index 1c70f1c75009..6c1fb6107bf8 100644 --- a/exporter/awsxrayexporter/internal/translator/span_links_test.go +++ b/exporter/awsxrayexporter/internal/translator/span_links_test.go @@ -5,7 +5,6 @@ package translator // import "github.com/open-telemetry/opentelemetry-collector- import ( "encoding/binary" - "strings" "testing" "time" @@ -37,10 +36,10 @@ func TestSpanLinkSimple(t *testing.T) { jsonStr, _ := MakeSegmentDocumentString(span, resource, nil, false, nil, false) - assert.True(t, strings.Contains(jsonStr, "links")) - assert.False(t, strings.Contains(jsonStr, "attributes")) - assert.True(t, strings.Contains(jsonStr, convertedTraceID)) - assert.True(t, strings.Contains(jsonStr, spanLink.SpanID().String())) + assert.Contains(t, jsonStr, "links") + assert.NotContains(t, jsonStr, "attributes") + assert.Contains(t, jsonStr, convertedTraceID) + assert.Contains(t, jsonStr, spanLink.SpanID().String()) } func TestSpanLinkEmpty(t *testing.T) { @@ -56,7 +55,7 @@ func TestSpanLinkEmpty(t *testing.T) { jsonStr, _ := MakeSegmentDocumentString(span, resource, nil, false, nil, false) - assert.False(t, strings.Contains(jsonStr, "links")) + assert.NotContains(t, jsonStr, "links") } func TestOldSpanLinkError(t *testing.T) { @@ -125,14 +124,14 @@ func TestTwoSpanLinks(t *testing.T) { jsonStr, _ := MakeSegmentDocumentString(span, resource, nil, false, nil, false) - assert.True(t, strings.Contains(jsonStr, "attributes")) - assert.True(t, strings.Contains(jsonStr, "links")) - assert.True(t, strings.Contains(jsonStr, "myKey1")) - assert.True(t, strings.Contains(jsonStr, "myKey2")) - assert.True(t, strings.Contains(jsonStr, "ABC")) - assert.True(t, strings.Contains(jsonStr, "1234")) - assert.True(t, strings.Contains(jsonStr, convertedTraceID1)) - assert.True(t, strings.Contains(jsonStr, convertedTraceID2)) + assert.Contains(t, jsonStr, "attributes") + assert.Contains(t, jsonStr, "links") + assert.Contains(t, jsonStr, "myKey1") + assert.Contains(t, jsonStr, "myKey2") + assert.Contains(t, jsonStr, "ABC") + assert.Contains(t, jsonStr, "1234") + assert.Contains(t, jsonStr, convertedTraceID1) + assert.Contains(t, jsonStr, convertedTraceID2) } func TestSpanLinkComplexAttributes(t *testing.T) { @@ -198,34 +197,34 @@ func TestSpanLinkComplexAttributes(t *testing.T) { jsonStr, _ := MakeSegmentDocumentString(span, resource, nil, false, nil, false) - assert.True(t, strings.Contains(jsonStr, "links")) + assert.Contains(t, jsonStr, "links") - assert.True(t, strings.Contains(jsonStr, "myKey1")) - assert.True(t, strings.Contains(jsonStr, "myValue")) + assert.Contains(t, jsonStr, "myKey1") + assert.Contains(t, jsonStr, "myValue") - assert.True(t, strings.Contains(jsonStr, "myKey2")) - assert.True(t, strings.Contains(jsonStr, "true")) + assert.Contains(t, jsonStr, "myKey2") + assert.Contains(t, jsonStr, "true") - assert.True(t, strings.Contains(jsonStr, "myKey3")) - assert.True(t, strings.Contains(jsonStr, "112233")) + assert.Contains(t, jsonStr, "myKey3") + assert.Contains(t, jsonStr, "112233") - assert.True(t, strings.Contains(jsonStr, "myKey4")) - assert.True(t, strings.Contains(jsonStr, "3.1415")) + assert.Contains(t, jsonStr, "myKey4") + assert.Contains(t, jsonStr, "3.1415") - assert.True(t, strings.Contains(jsonStr, "myKey5")) - assert.True(t, strings.Contains(jsonStr, "apple")) - assert.True(t, strings.Contains(jsonStr, "pear")) - assert.True(t, strings.Contains(jsonStr, "banana")) + assert.Contains(t, jsonStr, "myKey5") + assert.Contains(t, jsonStr, "apple") + assert.Contains(t, jsonStr, "pear") + assert.Contains(t, jsonStr, "banana") - assert.True(t, strings.Contains(jsonStr, "myKey6")) - assert.True(t, strings.Contains(jsonStr, "false")) + assert.Contains(t, jsonStr, "myKey6") + assert.Contains(t, jsonStr, "false") - assert.True(t, strings.Contains(jsonStr, "myKey7")) - assert.True(t, strings.Contains(jsonStr, "1234")) - assert.True(t, strings.Contains(jsonStr, "5678")) - assert.True(t, strings.Contains(jsonStr, "9012")) + assert.Contains(t, jsonStr, "myKey7") + assert.Contains(t, jsonStr, "1234") + assert.Contains(t, jsonStr, "5678") + assert.Contains(t, jsonStr, "9012") - assert.True(t, strings.Contains(jsonStr, "myKey8")) - assert.True(t, strings.Contains(jsonStr, "2.718")) - assert.True(t, strings.Contains(jsonStr, "1.618")) + assert.Contains(t, jsonStr, "myKey8") + assert.Contains(t, jsonStr, "2.718") + assert.Contains(t, jsonStr, "1.618") } diff --git a/exporter/awsxrayexporter/internal/translator/sql_test.go b/exporter/awsxrayexporter/internal/translator/sql_test.go index c15ecc639d87..cba2222b64d7 100644 --- a/exporter/awsxrayexporter/internal/translator/sql_test.go +++ b/exporter/awsxrayexporter/internal/translator/sql_test.go @@ -4,7 +4,6 @@ package translator import ( - "strings" "testing" "time" @@ -35,7 +34,7 @@ func TestClientSpanWithStatementAttribute(t *testing.T) { require.NoError(t, w.Encode(sqlData)) jsonStr := w.String() testWriters.release(w) - assert.True(t, strings.Contains(jsonStr, "mysql://db.example.com:3306/customers")) + assert.Contains(t, jsonStr, "mysql://db.example.com:3306/customers") } func TestClientSpanWithNonSQLDatabase(t *testing.T) { diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index 9a9b86a5be6f..32584fd7f8a6 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -233,7 +233,7 @@ func TestExporterLogs(t *testing.T) { t.Run("publish with dedup", func(t *testing.T) { rec := newBulkRecorder() server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { - assert.Equal(t, `{"@timestamp":"1970-01-01T00:00:00.000000000Z","Scope":{"name":"","value":"value","version":""},"SeverityNumber":0,"TraceFlags":0}`, string(docs[0].Document)) + assert.JSONEq(t, `{"@timestamp":"1970-01-01T00:00:00.000000000Z","Scope":{"name":"","value":"value","version":""},"SeverityNumber":0,"TraceFlags":0}`, string(docs[0].Document)) rec.Record(docs) return itemsAllOK(docs) }) @@ -706,9 +706,9 @@ func TestExporterLogs(t *testing.T) { assert.Len(t, rec.Items(), 1) doc := rec.Items()[0].Document - assert.Equal(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) - assert.Equal(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) - assert.Equal(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) + assert.JSONEq(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) + assert.JSONEq(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) t.Run("otel mode attribute key prefix conflict", func(t *testing.T) { @@ -1223,9 +1223,9 @@ func TestExporterMetrics(t *testing.T) { assert.Len(t, rec.Items(), 1) doc := rec.Items()[0].Document - assert.Equal(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) - assert.Equal(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) - assert.Equal(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) + assert.JSONEq(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) + assert.JSONEq(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) t.Run("otel mode _doc_count hint", func(t *testing.T) { @@ -1689,9 +1689,9 @@ func TestExporterTraces(t *testing.T) { assert.Len(t, rec.Items(), 2) for _, item := range rec.Items() { doc := item.Document - assert.Equal(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) - assert.Equal(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) - assert.Equal(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) + assert.JSONEq(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.JSONEq(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) + assert.JSONEq(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) } }) diff --git a/exporter/honeycombmarkerexporter/logs_exporter_test.go b/exporter/honeycombmarkerexporter/logs_exporter_test.go index 3eceacedaf4d..05f2e8c61f46 100644 --- a/exporter/honeycombmarkerexporter/logs_exporter_test.go +++ b/exporter/honeycombmarkerexporter/logs_exporter_test.go @@ -8,7 +8,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -138,7 +137,7 @@ func TestExportMarkers(t *testing.T) { userAgent := req.Header.Get(userAgentHeaderKey) assert.NotEmpty(t, userAgent) - assert.True(t, strings.Contains(userAgent, "OpenTelemetry Collector")) + assert.Contains(t, userAgent, "OpenTelemetry Collector") rw.WriteHeader(http.StatusAccepted) })) diff --git a/exporter/mezmoexporter/exporter_test.go b/exporter/mezmoexporter/exporter_test.go index 885dee55756d..868cdf8153bf 100644 --- a/exporter/mezmoexporter/exporter_test.go +++ b/exporter/mezmoexporter/exporter_test.go @@ -268,5 +268,5 @@ func Test404IngestError(t *testing.T) { responseField := logLine.Context[0] assert.Equal(t, "response", responseField.Key) - assert.Equal(t, `{"foo":"bar"}`, responseField.String) + assert.JSONEq(t, `{"foo":"bar"}`, responseField.String) } diff --git a/exporter/pulsarexporter/jaeger_marshaler_test.go b/exporter/pulsarexporter/jaeger_marshaler_test.go index 8b0eb9bec1ca..4df974bac0f2 100644 --- a/exporter/pulsarexporter/jaeger_marshaler_test.go +++ b/exporter/pulsarexporter/jaeger_marshaler_test.go @@ -42,7 +42,7 @@ func TestJaegerJsonBatchMarshaler(t *testing.T) { } jaegerJSONMessages, err := jaegerJSONMarshaler.Marshal(ptraces, "") require.NoError(t, err) - assert.Equal(t, jaegerJSONMessages[0].Payload, jsonBytes) + assert.JSONEq(t, string(jaegerJSONMessages[0].Payload), string(jsonBytes)) } func TestJaegerProtoBatchMarshaler(t *testing.T) { diff --git a/exporter/pulsarexporter/marshaler_test.go b/exporter/pulsarexporter/marshaler_test.go index b82e5531cae4..d84f69c89e02 100644 --- a/exporter/pulsarexporter/marshaler_test.go +++ b/exporter/pulsarexporter/marshaler_test.go @@ -102,7 +102,7 @@ func TestOTLPTracesJsonMarshaling(t *testing.T) { // Since marshaling json is not guaranteed to be in order // within a string, using a map to compare that the expected values are there - expectedJSON := map[string]any{ + expectedMap := map[string]any{ "resourceSpans": []any{ map[string]any{ "resource": map[string]any{}, @@ -133,5 +133,5 @@ func TestOTLPTracesJsonMarshaling(t *testing.T) { err = json.Unmarshal(payload, &final) require.NoError(t, err, "Must not error marshaling expected data") - assert.Equal(t, expectedJSON, final, "Must match the expected value") + assert.Equal(t, expectedMap, final, "Must match the expected value") } diff --git a/exporter/splunkhecexporter/metricdata_to_splunk_test.go b/exporter/splunkhecexporter/metricdata_to_splunk_test.go index a2ea1bb718d3..b15d5e840b0c 100644 --- a/exporter/splunkhecexporter/metricdata_to_splunk_test.go +++ b/exporter/splunkhecexporter/metricdata_to_splunk_test.go @@ -716,7 +716,7 @@ func TestMergeEvents(t *testing.T) { require.Len(t, merged, 1) b, err := json.Marshal(merged[0]) require.NoError(t, err) - require.Equal(t, `{"host":"","event":"metric","fields":{"IF-Azure":"azure-env","k8s.cluster.name":"devops-uat","k8s.namespace.name":"splunk-collector-tests","k8s.node.name":"myk8snodename","k8s.pod.name":"my-otel-collector-pod","metric_name:otel.collector.test":3411,"metric_name:otel.collector.test2":26059,"metric_type":"Gauge","metricsIndex":"test_metrics","metricsPlatform":"unset","resourceAttrs":"NO","testNumber":"number42","testRun":"42"}}`, string(b)) + require.JSONEq(t, `{"host":"","event":"metric","fields":{"IF-Azure":"azure-env","k8s.cluster.name":"devops-uat","k8s.namespace.name":"splunk-collector-tests","k8s.node.name":"myk8snodename","k8s.pod.name":"my-otel-collector-pod","metric_name:otel.collector.test":3411,"metric_name:otel.collector.test2":26059,"metric_type":"Gauge","metricsIndex":"test_metrics","metricsPlatform":"unset","resourceAttrs":"NO","testNumber":"number42","testRun":"42"}}`, string(b)) } func newMetricsWithResources() pcommon.Resource { diff --git a/exporter/syslogexporter/rfc5424_formatter_test.go b/exporter/syslogexporter/rfc5424_formatter_test.go index 3c6f48cd8ee7..edf4e157ffc8 100644 --- a/exporter/syslogexporter/rfc5424_formatter_test.go +++ b/exporter/syslogexporter/rfc5424_formatter_test.go @@ -6,7 +6,6 @@ package syslogexporter import ( "fmt" "regexp" - "strings" "testing" "time" @@ -81,10 +80,10 @@ func TestRFC5424Formatter(t *testing.T) { matched, err := regexp.MatchString(expectedRegex, actual) assert.NoError(t, err) assert.Truef(t, matched, "unexpected form of formatted message, formatted message: %s, regexp: %s", actual, expectedRegex) - assert.True(t, strings.Contains(actual, "Realm=\"SecureAuth0\"")) - assert.True(t, strings.Contains(actual, "UserHostAddress=\"192.168.2.132\"")) - assert.True(t, strings.Contains(actual, "UserID=\"Tester2\"")) - assert.True(t, strings.Contains(actual, "PEN=\"27389\"")) + assert.Contains(t, actual, "Realm=\"SecureAuth0\"") + assert.Contains(t, actual, "UserHostAddress=\"192.168.2.132\"") + assert.Contains(t, actual, "UserID=\"Tester2\"") + assert.Contains(t, actual, "PEN=\"27389\"") // Test defaults expected = "<165>1 2003-08-24T12:14:15.000003Z - - - - -\n" diff --git a/extension/encoding/avrologencodingextension/extension_test.go b/extension/encoding/avrologencodingextension/extension_test.go index c8b44021dc47..b66f12fbd9b0 100644 --- a/extension/encoding/avrologencodingextension/extension_test.go +++ b/extension/encoding/avrologencodingextension/extension_test.go @@ -34,7 +34,7 @@ func TestUnmarshal(t *testing.T) { logRecord := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0) assert.NoError(t, err) - assert.Equal(t, "{\"count\":5,\"hostname\":\"host1\",\"level\":\"warn\",\"levelEnum\":\"INFO\",\"mapField\":{},\"message\":\"log message\",\"nestedRecord\":{\"field1\":12,\"field2\":\"val2\"},\"properties\":[\"prop1\",\"prop2\"],\"severity\":1,\"timestamp\":1697187201488000000}", logRecord.Body().AsString()) + assert.JSONEq(t, "{\"count\":5,\"hostname\":\"host1\",\"level\":\"warn\",\"levelEnum\":\"INFO\",\"mapField\":{},\"message\":\"log message\",\"nestedRecord\":{\"field1\":12,\"field2\":\"val2\"},\"properties\":[\"prop1\",\"prop2\"],\"severity\":1,\"timestamp\":1697187201488000000}", logRecord.Body().AsString()) } func TestInvalidUnmarshal(t *testing.T) { diff --git a/extension/encoding/jsonlogencodingextension/json_test.go b/extension/encoding/jsonlogencodingextension/json_test.go index 2fcfa3d13129..afb77d24f5b4 100644 --- a/extension/encoding/jsonlogencodingextension/json_test.go +++ b/extension/encoding/jsonlogencodingextension/json_test.go @@ -25,7 +25,7 @@ func TestMarshalUnmarshal(t *testing.T) { buf, err := e.MarshalLogs(ld) assert.NoError(t, err) assert.NotEmpty(t, buf) - assert.Equal(t, json, string(buf)) + assert.JSONEq(t, json, string(buf)) } func TestInvalidMarshal(t *testing.T) { @@ -59,7 +59,7 @@ func TestPrettyLogProcessor(t *testing.T) { lp, err := j.logProcessor(sampleLog()) assert.NoError(t, err) assert.NotNil(t, lp) - assert.Equal(t, `[{"body":{"log":"test"},"logAttributes":{"foo":"bar"},"resourceAttributes":{"test":"logs-test"}},{"body":"log testing","resourceAttributes":{"test":"logs-test"}}]`, string(lp)) + assert.JSONEq(t, `[{"body":{"log":"test"},"logAttributes":{"foo":"bar"},"resourceAttributes":{"test":"logs-test"}},{"body":"log testing","resourceAttributes":{"test":"logs-test"}}]`, string(lp)) } func sampleLog() plog.Logs { diff --git a/extension/healthcheckextension/integration_test.go b/extension/healthcheckextension/integration_test.go index af8756b50892..ba765f569ab7 100644 --- a/extension/healthcheckextension/integration_test.go +++ b/extension/healthcheckextension/integration_test.go @@ -37,7 +37,7 @@ func Test_SimpleHealthCheck(t *testing.T) { var buf bytes.Buffer _, err = io.Copy(&buf, resp.Body) require.NoError(t, err) - assert.Equal(t, `{"status":"Server not available","upSince":"0001-01-01T00:00:00Z","uptime":""}`, buf.String()) + assert.JSONEq(t, `{"status":"Server not available","upSince":"0001-01-01T00:00:00Z","uptime":""}`, buf.String()) err = e.(*healthCheckExtension).Ready() require.NoError(t, err) resp, err = http.DefaultClient.Get(fmt.Sprintf("http://localhost:%d/", port)) diff --git a/extension/healthcheckv2extension/extension_test.go b/extension/healthcheckv2extension/extension_test.go index c15acc269245..57ba700e7e31 100644 --- a/extension/healthcheckv2extension/extension_test.go +++ b/extension/healthcheckv2extension/extension_test.go @@ -131,5 +131,5 @@ func TestNotifyConfig(t *testing.T) { body, err := io.ReadAll(resp.Body) require.NoError(t, err) - assert.Equal(t, confJSON, body) + assert.JSONEq(t, string(confJSON), string(body)) } diff --git a/extension/healthcheckv2extension/internal/http/server_test.go b/extension/healthcheckv2extension/internal/http/server_test.go index 4a11d0138d55..de365258206e 100644 --- a/extension/healthcheckv2extension/internal/http/server_test.go +++ b/extension/healthcheckv2extension/internal/http/server_test.go @@ -2986,7 +2986,7 @@ func TestStatus(t *testing.T) { body, err := io.ReadAll(resp.Body) require.NoError(t, err) - assert.True(t, strings.Contains(string(body), ts.expectedBody)) + assert.Contains(t, string(body), ts.expectedBody) if ts.expectedComponentStatus != nil { st := &serializableStatus{} diff --git a/extension/httpforwarderextension/extension_test.go b/extension/httpforwarderextension/extension_test.go index 7e3563bf6fe8..f5acd26cbe30 100644 --- a/extension/httpforwarderextension/extension_test.go +++ b/extension/httpforwarderextension/extension_test.go @@ -205,7 +205,7 @@ func TestExtension(t *testing.T) { if test.startUpError { err = hf.Start(ctx, componenttest.NewNopHost()) if test.startUpErrorMessage != "" { - require.True(t, strings.Contains(err.Error(), test.startUpErrorMessage)) + require.Contains(t, err.Error(), test.startUpErrorMessage) } require.Error(t, err) diff --git a/extension/httpforwarderextension/factory_test.go b/extension/httpforwarderextension/factory_test.go index 74824c3ab0ad..f515c94cdb47 100644 --- a/extension/httpforwarderextension/factory_test.go +++ b/extension/httpforwarderextension/factory_test.go @@ -5,7 +5,6 @@ package httpforwarderextension import ( "context" - "strings" "testing" "time" @@ -57,7 +56,7 @@ func TestFactory(t *testing.T) { ) if test.wantErr { if test.wantErrMessage != "" { - require.True(t, strings.Contains(err.Error(), test.wantErrMessage)) + require.Contains(t, err.Error(), test.wantErrMessage) } require.Error(t, err) require.Nil(t, e) diff --git a/extension/jaegerremotesampling/internal/http_test.go b/extension/jaegerremotesampling/internal/http_test.go index 0ad7a113e9a5..075073d82215 100644 --- a/extension/jaegerremotesampling/internal/http_test.go +++ b/extension/jaegerremotesampling/internal/http_test.go @@ -80,7 +80,7 @@ func TestEndpointsAreWired(t *testing.T) { resp.Body.Close() body := string(samplingStrategiesBytes) - assert.Equal(t, `{"probabilisticSampling":{"samplingRate":1}}`, body) + assert.JSONEq(t, `{"probabilisticSampling":{"samplingRate":1}}`, body) }) } } diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 373c22ae3427..84ff19abb0e5 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -1,9 +1,11 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/tools -go 1.22.0 +go 1.22.1 + +toolchain go1.22.8 require ( - github.com/Antonboom/testifylint v1.4.3 + github.com/Antonboom/testifylint v1.5.0 github.com/Khan/genqlient v0.7.0 github.com/client9/misspell v0.3.4 github.com/daixiang0/gci v0.13.5 diff --git a/internal/tools/go.sum b/internal/tools/go.sum index ff9fceffcdc2..b9dc248f5aab 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -12,8 +12,8 @@ github.com/Antonboom/errname v0.1.13 h1:JHICqsewj/fNckzrfVSe+T33svwQxmjC+1ntDsHO github.com/Antonboom/errname v0.1.13/go.mod h1:uWyefRYRN54lBg6HseYCFhs6Qjcy41Y3Jl/dVhA87Ns= github.com/Antonboom/nilnil v0.1.9 h1:eKFMejSxPSA9eLSensFmjW2XTgTwJMjZ8hUHtV4s/SQ= github.com/Antonboom/nilnil v0.1.9/go.mod h1:iGe2rYwCq5/Me1khrysB4nwI7swQvjclR8/YRPl5ihQ= -github.com/Antonboom/testifylint v1.4.3 h1:ohMt6AHuHgttaQ1xb6SSnxCeK4/rnK7KKzbvs7DmEck= -github.com/Antonboom/testifylint v1.4.3/go.mod h1:+8Q9+AOLsz5ZiQiiYujJKs9mNz398+M6UgslP4qgJLA= +github.com/Antonboom/testifylint v1.5.0 h1:dlUIsDMtCrZWUnvkaCz3quJCoIjaGi41GzjPBGkkJ8A= +github.com/Antonboom/testifylint v1.5.0/go.mod h1:wqaJbu0Blb5Wag2wv7Z5xt+CIV+eVLxtGZrlK13z3AE= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Crocmagnon/fatcontext v0.2.2 h1:OrFlsDdOj9hW/oBEJBNSuH7QWf+E9WPVHw+x52bXVbk= diff --git a/pkg/stanza/operator/operatortest/confmap.go b/pkg/stanza/operator/operatortest/confmap.go index 90486dd435e5..dc237dfaa6ce 100644 --- a/pkg/stanza/operator/operatortest/confmap.go +++ b/pkg/stanza/operator/operatortest/confmap.go @@ -4,7 +4,6 @@ package operatortest // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/operatortest" import ( - "fmt" "testing" "github.com/stretchr/testify/require" @@ -37,7 +36,7 @@ func (c ConfigUnmarshalTests) Run(t *testing.T) { t.Run(tc.Name, func(t *testing.T) { testConfMap, err := testConfMaps.Sub(tc.Name) require.NoError(t, err) - require.NotEmpty(t, testConfMap.AllKeys(), fmt.Sprintf("config not found: '%s'", tc.Name)) + require.NotEmpty(t, testConfMap.AllKeys(), "config not found: '%s'", tc.Name) cfg := newAnyOpConfig(c.DefaultConfig) err = testConfMap.Unmarshal(cfg) diff --git a/processor/remotetapprocessor/server_test.go b/processor/remotetapprocessor/server_test.go index aa27991ff900..8a6b68397e0f 100644 --- a/processor/remotetapprocessor/server_test.go +++ b/processor/remotetapprocessor/server_test.go @@ -50,7 +50,7 @@ func TestSocketConnectionLogs(t *testing.T) { n, _ := wsConn.Read(buf) return n == 132 }, 1*time.Second, 100*time.Millisecond, "received message") - require.Equal(t, `{"resourceLogs":[{"resource":{},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"stringValue":"foo"},"traceId":"","spanId":""}]}]}]}`, string(buf[0:132])) + require.JSONEq(t, `{"resourceLogs":[{"resource":{},"scopeLogs":[{"scope":{},"logRecords":[{"body":{"stringValue":"foo"},"traceId":"","spanId":""}]}]}]}`, string(buf[0:132])) err = processor.Shutdown(context.Background()) require.NoError(t, err) @@ -86,7 +86,7 @@ func TestSocketConnectionMetrics(t *testing.T) { n, _ := wsConn.Read(buf) return n == 94 }, 1*time.Second, 100*time.Millisecond, "received message") - require.Equal(t, `{"resourceMetrics":[{"resource":{},"scopeMetrics":[{"scope":{},"metrics":[{"name":"foo"}]}]}]}`, string(buf[0:94])) + require.JSONEq(t, `{"resourceMetrics":[{"resource":{},"scopeMetrics":[{"scope":{},"metrics":[{"name":"foo"}]}]}]}`, string(buf[0:94])) err = processor.Shutdown(context.Background()) require.NoError(t, err) @@ -122,7 +122,7 @@ func TestSocketConnectionTraces(t *testing.T) { n, _ := wsConn.Read(buf) return n == 143 }, 1*time.Second, 100*time.Millisecond, "received message") - require.Equal(t, `{"resourceSpans":[{"resource":{},"scopeSpans":[{"scope":{},"spans":[{"traceId":"","spanId":"","parentSpanId":"","name":"foo","status":{}}]}]}]}`, string(buf[0:143])) + require.JSONEq(t, `{"resourceSpans":[{"resource":{},"scopeSpans":[{"scope":{},"spans":[{"traceId":"","spanId":"","parentSpanId":"","name":"foo","status":{}}]}]}]}`, string(buf[0:143])) err = processor.Shutdown(context.Background()) require.NoError(t, err) diff --git a/processor/tailsamplingprocessor/internal/cache/lru_cache_test.go b/processor/tailsamplingprocessor/internal/cache/lru_cache_test.go index 36627bed0e87..b0f86e1c46a5 100644 --- a/processor/tailsamplingprocessor/internal/cache/lru_cache_test.go +++ b/processor/tailsamplingprocessor/internal/cache/lru_cache_test.go @@ -61,8 +61,8 @@ func TestLeastRecentlyUsedIsEvicted(t *testing.T) { c.Put(id1, true) c.Put(id2, true) v, ok := c.Get(id1) // use id1 - assert.True(t, true, v) - assert.True(t, true, ok) + assert.True(t, v) + assert.True(t, ok) c.Put(id3, true) v, ok = c.Get(id1) diff --git a/processor/transformprocessor/internal/metrics/func_convert_exponential_hist_to_explicit_hist_test.go b/processor/transformprocessor/internal/metrics/func_convert_exponential_hist_to_explicit_hist_test.go index 8d5d6ec8aa3c..f61969b17c18 100644 --- a/processor/transformprocessor/internal/metrics/func_convert_exponential_hist_to_explicit_hist_test.go +++ b/processor/transformprocessor/internal/metrics/func_convert_exponential_hist_to_explicit_hist_test.go @@ -4,7 +4,6 @@ package metrics import ( - "strings" "testing" "time" @@ -782,7 +781,7 @@ func Test_convertExponentialHistToExplicitHist_validate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { _, err := convertExponentialHistToExplicitHist("random", tt.sliceExplicitBoundsArgs) assert.Error(t, err) - assert.True(t, strings.Contains(err.Error(), "explicit bounds cannot be empty")) + assert.Contains(t, err.Error(), "explicit bounds cannot be empty") }) } } diff --git a/receiver/hostmetricsreceiver/internal/scraper/processscraper/internal/handlecount/handles_windows_test.go b/receiver/hostmetricsreceiver/internal/scraper/processscraper/internal/handlecount/handles_windows_test.go index 9ce32bf51e51..b757c83404bb 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/processscraper/internal/handlecount/handles_windows_test.go +++ b/receiver/hostmetricsreceiver/internal/scraper/processscraper/internal/handlecount/handles_windows_test.go @@ -7,7 +7,6 @@ package handlecount import ( "errors" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -33,7 +32,7 @@ func TestHandleCountManager(t *testing.T) { _, err = m.GetProcessHandleCount(3) assert.ErrorIs(t, errors.Unwrap(err), ErrNoHandleCountForProcess) - assert.True(t, strings.Contains(err.Error(), "3")) + assert.Contains(t, err.Error(), "3") } type mockQueryer struct { diff --git a/receiver/splunkhecreceiver/receiver_test.go b/receiver/splunkhecreceiver/receiver_test.go index 3c7351aedd0e..f45b3ee2e011 100644 --- a/receiver/splunkhecreceiver/receiver_test.go +++ b/receiver/splunkhecreceiver/receiver_test.go @@ -41,14 +41,14 @@ import ( func assertHecSuccessResponse(t *testing.T, resp *http.Response, body any) { status := resp.StatusCode assert.Equal(t, http.StatusOK, status) - assert.Equal(t, httpJSONTypeHeader, resp.Header.Get(httpContentTypeHeader)) + assert.Equal(t, "application/json", resp.Header.Get(httpContentTypeHeader)) assert.Equal(t, map[string]any{"code": float64(0), "text": "Success"}, body) } func assertHecSuccessResponseWithAckID(t *testing.T, resp *http.Response, body any, ackID uint64) { status := resp.StatusCode assert.Equal(t, http.StatusOK, status) - assert.Equal(t, httpJSONTypeHeader, resp.Header.Get(httpContentTypeHeader)) + assert.Equal(t, "application/json", resp.Header.Get(httpContentTypeHeader)) assert.Equal(t, map[string]any{"code": float64(0), "text": "Success", "ackId": float64(ackID)}, body) } @@ -1768,7 +1768,7 @@ func Test_splunkhecreceiver_handleHealthPath(t *testing.T) { respBytes, err := io.ReadAll(resp.Body) assert.NoError(t, err) defer resp.Body.Close() - assert.Equal(t, responseHecHealthy, string(respBytes)) + assert.JSONEq(t, responseHecHealthy, string(respBytes)) assert.Equal(t, 200, resp.StatusCode) } @@ -1837,7 +1837,7 @@ func Test_splunkhecreceiver_handle_nested_fields(t *testing.T) { assert.Equal(t, 1, sink.LogRecordCount()) } else { assert.Equal(t, http.StatusBadRequest, w.Code) - assert.Equal(t, fmt.Sprintf(responseErrHandlingIndexedFields, 0), w.Body.String()) + assert.JSONEq(t, fmt.Sprintf(responseErrHandlingIndexedFields, 0), w.Body.String()) } }) @@ -1861,7 +1861,7 @@ func Test_splunkhecReceiver_rawReqHasmetadataInResource(t *testing.T) { assertResponse := func(t *testing.T, status int, body string) { assert.Equal(t, http.StatusOK, status) - assert.Equal(t, responseOK, body) + assert.JSONEq(t, responseOK, body) } tests := []struct { @@ -2012,7 +2012,7 @@ func Test_splunkhecReceiver_healthCheck_success(t *testing.T) { }(), assertResponse: func(t *testing.T, status int, body string) { assert.Equal(t, http.StatusOK, status) - assert.Equal(t, responseHecHealthy, body) + assert.JSONEq(t, responseHecHealthy, body) }, }, { @@ -2023,7 +2023,7 @@ func Test_splunkhecReceiver_healthCheck_success(t *testing.T) { }(), assertResponse: func(t *testing.T, status int, body string) { assert.Equal(t, http.StatusOK, status) - assert.Equal(t, responseHecHealthy, body) + assert.JSONEq(t, responseHecHealthy, body) }, }, { @@ -2034,7 +2034,7 @@ func Test_splunkhecReceiver_healthCheck_success(t *testing.T) { }(), assertResponse: func(t *testing.T, status int, body string) { assert.Equal(t, http.StatusBadRequest, status) - assert.Equal(t, responseNoData, body) + assert.JSONEq(t, responseNoData, body) }, }, } From 1ef15b92559b6f40a42ba32f3ac861b47271fa9d Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Mon, 21 Oct 2024 04:39:37 +0100 Subject: [PATCH 19/21] [exporter/elasticsearch] Enable gzip compression by default (#35865) #### Description Enable gzip compression by default, at hardcoded level BestSpeed. To disable compression, set `compression` to `none`. #### Link to tracking issue #### Testing #### Documentation --- ...lasticsearchexporter_compression-gzip.yaml | 27 ++++++ exporter/elasticsearchexporter/README.md | 2 +- exporter/elasticsearchexporter/bulkindexer.go | 7 ++ .../elasticsearchexporter/bulkindexer_test.go | 94 ++++++++++++++++--- exporter/elasticsearchexporter/config.go | 6 +- exporter/elasticsearchexporter/config_test.go | 30 +++++- exporter/elasticsearchexporter/esclient.go | 10 +- exporter/elasticsearchexporter/factory.go | 2 + exporter/elasticsearchexporter/go.mod | 2 +- .../testdata/config.yaml | 6 ++ exporter/elasticsearchexporter/utils_test.go | 7 +- 11 files changed, 169 insertions(+), 24 deletions(-) create mode 100644 .chloggen/elasticsearchexporter_compression-gzip.yaml diff --git a/.chloggen/elasticsearchexporter_compression-gzip.yaml b/.chloggen/elasticsearchexporter_compression-gzip.yaml new file mode 100644 index 000000000000..207d8f40322a --- /dev/null +++ b/.chloggen/elasticsearchexporter_compression-gzip.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Enable gzip compression by default + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35865] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: To disable compression, set config `compression` to `none`. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index b620b81158e9..eadb1e309803 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -71,7 +71,7 @@ service: ### HTTP settings -The Elasticsearch exporter supports common [HTTP Configuration Settings][confighttp], except for `compression` (all requests are uncompressed). +The Elasticsearch exporter supports common [HTTP Configuration Settings][confighttp]. Gzip compression is enabled by default. To disable compression, set `compression` to `none`. As a consequence of supporting [confighttp], the Elasticsearch exporter also supports common [TLS Configuration Settings][configtls]. The Elasticsearch exporter sets `timeout` (HTTP request timeout) to 90s by default. diff --git a/exporter/elasticsearchexporter/bulkindexer.go b/exporter/elasticsearchexporter/bulkindexer.go index 21b48814914d..1700b03619d6 100644 --- a/exporter/elasticsearchexporter/bulkindexer.go +++ b/exporter/elasticsearchexporter/bulkindexer.go @@ -4,6 +4,7 @@ package elasticsearchexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter" import ( + "compress/gzip" "context" "errors" "io" @@ -15,6 +16,7 @@ import ( "github.com/elastic/go-docappender/v2" "github.com/elastic/go-elasticsearch/v7" + "go.opentelemetry.io/collector/config/configcompression" "go.uber.org/zap" ) @@ -68,12 +70,17 @@ func bulkIndexerConfig(client *elasticsearch.Client, config *Config) docappender maxDocRetries = config.Retry.MaxRetries } } + var compressionLevel int + if config.Compression == configcompression.TypeGzip { + compressionLevel = gzip.BestSpeed + } return docappender.BulkIndexerConfig{ Client: client, MaxDocumentRetries: maxDocRetries, Pipeline: config.Pipeline, RetryOnDocumentStatus: config.Retry.RetryOnStatus, RequireDataStream: config.MappingMode() == MappingOTel, + CompressionLevel: compressionLevel, } } diff --git a/exporter/elasticsearchexporter/bulkindexer_test.go b/exporter/elasticsearchexporter/bulkindexer_test.go index 7a75c6f5a0f1..194890e69aa6 100644 --- a/exporter/elasticsearchexporter/bulkindexer_test.go +++ b/exporter/elasticsearchexporter/bulkindexer_test.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/go-elasticsearch/v7" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/config/confighttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest/observer" @@ -62,13 +63,8 @@ func TestAsyncBulkIndexer_flushOnClose(t *testing.T) { }}) require.NoError(t, err) - bulkIndexer, err := newAsyncBulkIndexer(zap.NewNop(), client, &cfg) - require.NoError(t, err) - session, err := bulkIndexer.StartSession(context.Background()) - require.NoError(t, err) + bulkIndexer := runBulkIndexerOnce(t, &cfg, client) - assert.NoError(t, session.Add(context.Background(), "foo", strings.NewReader(`{"foo": "bar"}`), nil)) - assert.NoError(t, bulkIndexer.Close(context.Background())) assert.Equal(t, int64(1), bulkIndexer.stats.docsIndexed.Load()) } @@ -157,13 +153,7 @@ func TestAsyncBulkIndexer_requireDataStream(t *testing.T) { }}) require.NoError(t, err) - bulkIndexer, err := newAsyncBulkIndexer(zap.NewNop(), client, &tt.config) - require.NoError(t, err) - session, err := bulkIndexer.StartSession(context.Background()) - require.NoError(t, err) - - assert.NoError(t, session.Add(context.Background(), "foo", strings.NewReader(`{"foo": "bar"}`), nil)) - assert.NoError(t, bulkIndexer.Close(context.Background())) + runBulkIndexerOnce(t, &tt.config, client) assert.Equal(t, tt.wantRequireDataStream, <-requireDataStreamCh) }) @@ -234,6 +224,8 @@ func TestAsyncBulkIndexer_flush_error(t *testing.T) { bulkIndexer, err := newAsyncBulkIndexer(zap.New(core), client, &cfg) require.NoError(t, err) + defer bulkIndexer.Close(context.Background()) + session, err := bulkIndexer.StartSession(context.Background()) require.NoError(t, err) @@ -241,7 +233,6 @@ func TestAsyncBulkIndexer_flush_error(t *testing.T) { // should flush time.Sleep(100 * time.Millisecond) assert.Equal(t, int64(0), bulkIndexer.stats.docsIndexed.Load()) - assert.NoError(t, bulkIndexer.Close(context.Background())) messages := observed.FilterMessage(tt.wantMessage) require.Equal(t, 1, messages.Len(), "message not found; observed.All()=%v", observed.All()) for _, wantField := range tt.wantFields { @@ -250,3 +241,78 @@ func TestAsyncBulkIndexer_flush_error(t *testing.T) { }) } } + +func TestAsyncBulkIndexer_logRoundTrip(t *testing.T) { + tests := []struct { + name string + config Config + }{ + { + name: "compression none", + config: Config{ + NumWorkers: 1, + ClientConfig: confighttp.ClientConfig{Compression: "none"}, + }, + }, + { + name: "compression gzip", + config: Config{ + NumWorkers: 1, + ClientConfig: confighttp.ClientConfig{Compression: "gzip"}, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + loggerCore, logObserver := observer.New(zap.DebugLevel) + + esLogger := clientLogger{ + Logger: zap.New(loggerCore), + logRequestBody: true, + logResponseBody: true, + } + + client, err := elasticsearch.NewClient(elasticsearch.Config{ + Transport: &mockTransport{ + RoundTripFunc: func(*http.Request) (*http.Response, error) { + return &http.Response{ + Header: http.Header{"X-Elastic-Product": []string{"Elasticsearch"}}, + Body: io.NopCloser(strings.NewReader(successResp)), + }, nil + }, + }, + Logger: &esLogger, + }) + require.NoError(t, err) + + runBulkIndexerOnce(t, &tt.config, client) + + records := logObserver.AllUntimed() + assert.Len(t, records, 2) + + assert.Equal(t, "/", records[0].ContextMap()["path"]) + assert.Nil(t, records[0].ContextMap()["request_body"]) + assert.JSONEq(t, successResp, records[0].ContextMap()["response_body"].(string)) + + assert.Equal(t, "/_bulk", records[1].ContextMap()["path"]) + assert.Equal(t, "{\"create\":{\"_index\":\"foo\"}}\n{\"foo\": \"bar\"}\n", records[1].ContextMap()["request_body"]) + assert.JSONEq(t, successResp, records[1].ContextMap()["response_body"].(string)) + }) + } +} + +func runBulkIndexerOnce(t *testing.T, config *Config, client *elasticsearch.Client) *asyncBulkIndexer { + bulkIndexer, err := newAsyncBulkIndexer(zap.NewNop(), client, config) + require.NoError(t, err) + session, err := bulkIndexer.StartSession(context.Background()) + require.NoError(t, err) + + assert.NoError(t, session.Add(context.Background(), "foo", strings.NewReader(`{"foo": "bar"}`), nil)) + assert.NoError(t, bulkIndexer.Close(context.Background())) + + return bulkIndexer +} diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index fe794d6db430..0835396d928f 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "go.opentelemetry.io/collector/config/configcompression" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/exporter/exporterbatcher" @@ -273,9 +274,8 @@ func (cfg *Config) Validate() error { return fmt.Errorf("unknown mapping mode %q", cfg.Mapping.Mode) } - if cfg.Compression != "" { - // TODO support confighttp.ClientConfig.Compression - return errors.New("compression is not currently configurable") + if cfg.Compression != "none" && cfg.Compression != configcompression.TypeGzip { + return errors.New("compression must be one of [none, gzip]") } if cfg.Retry.MaxRequests != 0 && cfg.Retry.MaxRetries != 0 { diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index 9934dbb7365b..baec2bd9646a 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -6,6 +6,7 @@ package elasticsearchexporter import ( "net/http" "path/filepath" + "strings" "testing" "time" @@ -38,6 +39,7 @@ func TestConfig(t *testing.T) { defaultMaxIdleConns := 100 defaultIdleConnTimeout := 90 * time.Second + defaultCompression := configcompression.TypeGzip tests := []struct { configFile string @@ -80,6 +82,7 @@ func TestConfig(t *testing.T) { cfg.Headers = map[string]configopaque.String{ "myheader": "test", } + cfg.Compression = defaultCompression }), Authentication: AuthenticationSettings{ User: "elastic", @@ -150,6 +153,7 @@ func TestConfig(t *testing.T) { cfg.Headers = map[string]configopaque.String{ "myheader": "test", } + cfg.Compression = defaultCompression }), Authentication: AuthenticationSettings{ User: "elastic", @@ -220,6 +224,7 @@ func TestConfig(t *testing.T) { cfg.Headers = map[string]configopaque.String{ "myheader": "test", } + cfg.Compression = defaultCompression }), Authentication: AuthenticationSettings{ User: "elastic", @@ -301,10 +306,29 @@ func TestConfig(t *testing.T) { cfg.Batcher.Enabled = &enabled }), }, + { + id: component.NewIDWithName(metadata.Type, "compression_none"), + configFile: "config.yaml", + expected: withDefaultConfig(func(cfg *Config) { + cfg.Endpoint = "https://elastic.example.com:9200" + + cfg.Compression = "none" + }), + }, + { + id: component.NewIDWithName(metadata.Type, "compression_gzip"), + configFile: "config.yaml", + expected: withDefaultConfig(func(cfg *Config) { + cfg.Endpoint = "https://elastic.example.com:9200" + + cfg.Compression = "gzip" + }), + }, } for _, tt := range tests { - t.Run(tt.id.String(), func(t *testing.T) { + tt := tt + t.Run(strings.ReplaceAll(tt.id.String(), "/", "_"), func(t *testing.T) { factory := NewFactory() cfg := factory.CreateDefaultConfig() @@ -387,9 +411,9 @@ func TestConfig_Validate(t *testing.T) { "compression unsupported": { config: withDefaultConfig(func(cfg *Config) { cfg.Endpoints = []string{"http://test:9200"} - cfg.Compression = configcompression.TypeGzip + cfg.Compression = configcompression.TypeSnappy }), - err: `compression is not currently configurable`, + err: `compression must be one of [none, gzip]`, }, "both max_retries and max_requests specified": { config: withDefaultConfig(func(cfg *Config) { diff --git a/exporter/elasticsearchexporter/esclient.go b/exporter/elasticsearchexporter/esclient.go index 556718242bbf..927f62844ec3 100644 --- a/exporter/elasticsearchexporter/esclient.go +++ b/exporter/elasticsearchexporter/esclient.go @@ -11,6 +11,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/elastic/go-elasticsearch/v7" + "github.com/klauspost/compress/gzip" "go.opentelemetry.io/collector/component" "go.uber.org/zap" @@ -32,7 +33,14 @@ func (cl *clientLogger) LogRoundTrip(requ *http.Request, resp *http.Response, cl var fields []zap.Field if cl.logRequestBody && requ != nil && requ.Body != nil { - if b, err := io.ReadAll(requ.Body); err == nil { + body := requ.Body + if requ.Header.Get("Content-Encoding") == "gzip" { + if r, err := gzip.NewReader(body); err == nil { + defer r.Close() + body = r + } + } + if b, err := io.ReadAll(body); err == nil { fields = append(fields, zap.ByteString("request_body", b)) } } diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 61af38d5cee6..31f9a17895f7 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -11,6 +11,7 @@ import ( "time" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configcompression" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/exporter" @@ -44,6 +45,7 @@ func createDefaultConfig() component.Config { httpClientConfig := confighttp.NewDefaultClientConfig() httpClientConfig.Timeout = 90 * time.Second + httpClientConfig.Compression = configcompression.TypeGzip return &Config{ QueueSettings: qs, diff --git a/exporter/elasticsearchexporter/go.mod b/exporter/elasticsearchexporter/go.mod index c9a675710622..470e56ec3e4d 100644 --- a/exporter/elasticsearchexporter/go.mod +++ b/exporter/elasticsearchexporter/go.mod @@ -8,6 +8,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/elastic/go-structform v0.0.12 github.com/json-iterator/go v1.1.12 + github.com/klauspost/compress v1.17.10 github.com/lestrrat-go/strftime v1.1.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.111.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.111.0 @@ -44,7 +45,6 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect - github.com/klauspost/compress v1.17.10 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.1.1 // indirect diff --git a/exporter/elasticsearchexporter/testdata/config.yaml b/exporter/elasticsearchexporter/testdata/config.yaml index d76d300a51c1..e3f7ffc67fa9 100644 --- a/exporter/elasticsearchexporter/testdata/config.yaml +++ b/exporter/elasticsearchexporter/testdata/config.yaml @@ -86,3 +86,9 @@ elasticsearch/batcher_disabled: endpoint: https://elastic.example.com:9200 batcher: enabled: false +elasticsearch/compression_none: + endpoint: https://elastic.example.com:9200 + compression: none +elasticsearch/compression_gzip: + endpoint: https://elastic.example.com:9200 + compression: gzip diff --git a/exporter/elasticsearchexporter/utils_test.go b/exporter/elasticsearchexporter/utils_test.go index 05e9d8576a57..94c475219ffb 100644 --- a/exporter/elasticsearchexporter/utils_test.go +++ b/exporter/elasticsearchexporter/utils_test.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/klauspost/compress/gzip" "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" @@ -160,7 +161,11 @@ func newESTestServer(t *testing.T, bulkHandler bulkHandler) *httptest.Server { tsStart := time.Now() var items []itemRequest - dec := json.NewDecoder(req.Body) + body := req.Body + if req.Header.Get("Content-Encoding") == "gzip" { + body, _ = gzip.NewReader(req.Body) + } + dec := json.NewDecoder(body) for dec.More() { var action, doc json.RawMessage if err := dec.Decode(&action); err != nil { From 79926f8647a9839cb2020aeddbcc33a248ed7422 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Mon, 21 Oct 2024 14:22:49 -0400 Subject: [PATCH 20/21] [chore] skip encoded-compare in testifylint (#35900) --- .golangci.yml | 1 + Makefile.Common | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index bf8564d6a445..e7d4a3846078 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -137,6 +137,7 @@ linters-settings: - float-compare - require-error - suite-subtest-run + - encoded-compare # has false positives that cannot be fixed with testifylint-fix enable-all: true linters: diff --git a/Makefile.Common b/Makefile.Common index f73df4c03097..07ce7e0d9062 100644 --- a/Makefile.Common +++ b/Makefile.Common @@ -78,7 +78,7 @@ GOTESTSUM := $(TOOLS_BIN_DIR)/gotestsum TESTIFYLINT := $(TOOLS_BIN_DIR)/testifylint GOTESTSUM_OPT?= --rerun-fails=1 -TESTIFYLINT_OPT?= --enable-all --disable=float-compare,require-error,suite-subtest-run +TESTIFYLINT_OPT?= --enable-all --disable=float-compare,require-error,suite-subtest-run,encoded-compare # BUILD_TYPE should be one of (dev, release). BUILD_TYPE?=release From 7d92d6a5c655d095a3ed16daca25ba756c7713f0 Mon Sep 17 00:00:00 2001 From: Jade Guiton Date: Tue, 22 Oct 2024 05:20:49 +0200 Subject: [PATCH 21/21] [chore] Fix `make update-otel` on macOS (#35587) **Description:** When working on #35580, it seemed that `make update-otel` was not properly updating the `builder-config.yaml` files. I traced this down to the `updatehelper` script not working as intended. This turned out to be because it uses some features of the `sed` utility which do not exist on macOS, causing no updates to be made. The `\s` (whitespace) regex class is a GNU extension for `sed`; the POSIX-compatible equivalent is `[[:space:]]` ([related SO question](https://stackoverflow.com/questions/18840175/find-and-replace-with-spaces-using-sed-mac-terminal)). Moreover, on macOS, the `-i` (in-place) option requires an argument, even if empty; otherwise the `-e` following it is parsed as that argument, creating an unnecessary backup file ([related SO question](https://stackoverflow.com/questions/4247068/sed-command-with-i-option-failing-on-mac-but-works-on-linux)). **Testing:** I manually tested this change as part of developing the aforementioned PR; the script seems to work on macOS now. It would be good for someone to test this change on Linux to make sure nothing has broken there. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1e087f09fea2..86ecba3a3276 100644 --- a/Makefile +++ b/Makefile @@ -395,16 +395,16 @@ define updatehelper echo "Usage: updatehelper "; \ exit 1; \ fi - grep "go\.opentelemetry\.io" $(1) | sed 's/^\s*-\s*//' | while IFS= read -r line; do \ + grep "go\.opentelemetry\.io" $(1) | sed 's/^[[:space:]]*-[[:space:]]*//' | while IFS= read -r line; do \ if grep -qF "$$line" $(2); then \ package=$$(grep -F "$$line" $(2) | head -n 1 | awk '{print $$1}'); \ version=$$(grep -F "$$line" $(2) | head -n 1 | awk '{print $$2}'); \ builder_package=$$(grep -F "$$package" $(3) | awk '{print $$3}'); \ builder_version=$$(grep -F "$$package" $(3) | awk '{print $$4}'); \ if [ "$$builder_package" == "$$package" ]; then \ - echo "$$builder_version";\ - sed -i -e "s|$$builder_package.*$$builder_version|$$builder_package $$version|" $(3); \ - echo "[$(3)]: $$package updated to $$version"; \ + sed -i.bak -e "s|$$builder_package.*$$builder_version|$$builder_package $$version|" $(3); \ + rm $(3).bak; \ + echo "[$(3)]: $$package updated from $$builder_version to $$version"; \ fi; \ fi; \ done