From 57a0bcb9e7f4a5874a2c71ebf07a043379566921 Mon Sep 17 00:00:00 2001 From: Jason Polanco Date: Thu, 21 Sep 2023 15:57:50 -0400 Subject: [PATCH 01/38] Migrate awsapmprocessor to main-apm (#502) Co-authored-by: Lisa Guo Co-authored-by: Ping Xiang <64551395+pxaws@users.noreply.github.com> Co-authored-by: nanzhenAWS <133444984+nanzhenAWS@users.noreply.github.com> --- go.mod | 1 + go.sum | 21 +- processor/awsapmprocessor/README.md | 130 ++ processor/awsapmprocessor/config.go | 18 + processor/awsapmprocessor/config_test.go | 104 ++ processor/awsapmprocessor/factory.go | 96 ++ .../internal/customconfiguration/common.go | 116 ++ .../internal/customconfiguration/dropper.go | 35 + .../customconfiguration/dropper_test.go | 185 +++ .../internal/customconfiguration/keeper.go | 35 + .../customconfiguration/keeper_test.go | 178 +++ .../internal/customconfiguration/replacer.go | 54 + .../customconfiguration/replacer_test.go | 281 +++++ .../normalizer/attributesnormalizer.go | 107 ++ .../normalizer/attributesnormalizer_test.go | 105 ++ .../internal/resolver/attributesresolver.go | 52 + .../resolver/attributesresolver_test.go | 67 + .../awsapmprocessor/internal/resolver/eks.go | 709 +++++++++++ .../internal/resolver/eks_test.go | 1114 +++++++++++++++++ processor/awsapmprocessor/processor.go | 268 ++++ .../awsapmprocessor/testdata/config.yaml | 29 + 21 files changed, 3696 insertions(+), 9 deletions(-) create mode 100644 processor/awsapmprocessor/README.md create mode 100644 processor/awsapmprocessor/config.go create mode 100644 processor/awsapmprocessor/config_test.go create mode 100644 processor/awsapmprocessor/factory.go create mode 100644 processor/awsapmprocessor/internal/customconfiguration/common.go create mode 100644 processor/awsapmprocessor/internal/customconfiguration/dropper.go create mode 100644 processor/awsapmprocessor/internal/customconfiguration/dropper_test.go create mode 100644 processor/awsapmprocessor/internal/customconfiguration/keeper.go create mode 100644 processor/awsapmprocessor/internal/customconfiguration/keeper_test.go create mode 100644 processor/awsapmprocessor/internal/customconfiguration/replacer.go create mode 100644 processor/awsapmprocessor/internal/customconfiguration/replacer_test.go create mode 100644 processor/awsapmprocessor/internal/normalizer/attributesnormalizer.go create mode 100644 processor/awsapmprocessor/internal/normalizer/attributesnormalizer_test.go create mode 100644 processor/awsapmprocessor/internal/resolver/attributesresolver.go create mode 100644 processor/awsapmprocessor/internal/resolver/attributesresolver_test.go create mode 100644 processor/awsapmprocessor/internal/resolver/eks.go create mode 100644 processor/awsapmprocessor/internal/resolver/eks_test.go create mode 100644 processor/awsapmprocessor/processor.go create mode 100644 processor/awsapmprocessor/testdata/config.yaml diff --git a/go.mod b/go.mod index 17f2bd8250..4b52f38a61 100644 --- a/go.mod +++ b/go.mod @@ -94,6 +94,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/eks v1.27.15 github.com/aws/smithy-go v1.13.5 github.com/bigkevmcd/go-configparser v0.0.0-20200217161103-d137835d2579 + github.com/deckarep/golang-set/v2 v2.3.1 github.com/go-kit/log v0.2.1 github.com/gobwas/glob v0.2.3 github.com/google/cadvisor v0.47.3 // indirect diff --git a/go.sum b/go.sum index 70d82c7d3a..3f509b0aff 100644 --- a/go.sum +++ b/go.sum @@ -25,27 +25,28 @@ cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aD cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.53.0 h1:K3wLbjbnSlxhuG5q4pntHv5AEbQM1QqHKGYgwFIqOTg= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 h1:aRVqY1p2IJaBGStWMsQMpkAa83cPkCDLl80eOj0Rbz4= -cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68/go.mod h1:1a3eRNYX12fs5UABBIXS8HXVvQbX9hRB/RkEBPORpe8= +cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= +cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= -cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/monitoring v1.13.0 h1:2qsrgXGVoRXpP7otZ14eE1I568zAa92sJSDPyOJvwjM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.30.0 h1:vCge8m7aUKBJYOgrZp7EsNDf6QMd2CAlXZqWTn3yq6s= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -346,6 +347,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= +github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/denisenkom/go-mssqldb v0.12.0 h1:VtrkII767ttSPNRfFekePK3sctr+joXgO58stqQbtUA= github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= diff --git a/processor/awsapmprocessor/README.md b/processor/awsapmprocessor/README.md new file mode 100644 index 0000000000..e380b3b912 --- /dev/null +++ b/processor/awsapmprocessor/README.md @@ -0,0 +1,130 @@ +# AWS APM Processor for Amazon Cloudwatch Agent + +The AWS APM processor is used to reduce the cardinality of telemtry metrics and traces before exporting them to CloudWatch Logs via [EMF](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awsemfexporter) and [X-Ray](github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter) respectively. +It reduces the cardinality of metrics/traces via 3 types of actions, `keep`, `drop` and `replace`, which are configured by users. CloudWatch Agent(CWA) customers will configure these rules with their CWA configurations, which will get translated to yaml. + +Note: Traces support only `replace` actions and are implicitly pulled from the logs section of the CWA configuration + +| Status | | +| ------------------------ |---------------------------| +| Stability | [beta] | +| Supported pipeline types | metrics, traces | +| Distributions | [amazon-cloudwatch-agent] | + +## Exporter Configuration + +The following exporter configuration parameters are supported. + +| Name | Description | Default | +|:---------------------------------------------|:------------------------------------------------------------------------------------------------------------------|---------| +| `resolvers` | Platform processor is being configured for. Currently supports EKS. EC2 platform will be supported in the future. | [eks] | +| `rules` | Custom configuration rules used for filtering metrics/traces. Can be of type `drop`, `keep`, `replace`. | [] | + +### rules +The rules section defines the rules (filters) to be applied + +| Name | Description | Default | +|:---------------|:-------------------------------------------------------------------------------------------------------------------------| --- | +| `selectors` | List of metrics/traces dimension matchers. | [] | +| `action` | Action being applied for the specified selector. `keep`, `drop`, `replace` | "" | +| `rule_name` | (Optional) Name of rule. | [] | +| `replacements` | (Optional) List of metrics/traces replacements to be executed. Based on specified selectors. requires `action = replace` | [] | + +#### selectors +A selectors section defines a matching against the dimensions of incoming metrics/traces. + +| Name | Description | Default | +|:------------|:--------------------------------------------------------------| ------ | +| `dimension` | Dimension of metrics/traces | "" | +| `match` | glob used for matching values of dimensions | "" | + +### replacements +A replacements section defines a matching against the dimensions of incoming metrics/traces for which value replacements will be done. action must be `replace` + +| Name | Description | Default | +|:-------------------|:----------------------------------------------| ------ | +| `target_dimension` | Dimension to replace | "" | +| `value` | Value to replace current dimension value with | "" | + + +## AWS APM Processor Configuration Example + +```yaml +awsapm: + resolvers: ["eks"] + rules: + - selectors: + - dimension: Operation + match: "POST *" + - dimension: RemoteService + match: "*" + action: keep + rule_name: "keep01" + - selectors: + - dimension: Operation + match: "GET *" + - dimension: RemoteService + match: "*" + action: keep + rule_name: "keep02" + - selectors: + - dimension: Operation + match: "POST *" + action: drop + rule_name: "drop01" + - selectors: + - dimension: Operation + match: "*" + replacements: + - target_dimension: RemoteOperation + value: "This is a test string" + action: replace + rule_name: "replace01" +``` + +## Amazon CloudWatch Agent Configuration Example + +```json +{ + "agent": { + "region": "us-west-2", + "debug": true + }, + "traces": { + "traces_collected": { + "apm": {} + } + }, + "logs": { + "metrics_collected": { + "apm": { + "rules": [ + { + "selectors": [ + { + "dimension": "Service", + "match": "pet-clinic-frontend" + }, + { + "dimension": "RemoteService", + "match": "customers-service" + } + ], + "action": "keep", + "rule_name": "keep01" + }, + { + "selectors": [ + { + "dimension": "Operation", + "match": "GET *" + } + ], + "action": "drop", + "rule_name": "drop01" + } + } + } + } + } +``` \ No newline at end of file diff --git a/processor/awsapmprocessor/config.go b/processor/awsapmprocessor/config.go new file mode 100644 index 0000000000..74772b1195 --- /dev/null +++ b/processor/awsapmprocessor/config.go @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsapmprocessor + +import ( + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/customconfiguration" +) + +type Config struct { + Resolvers []string `mapstructure:"resolvers"` + Rules []customconfiguration.Rule `mapstructure:"rules"` +} + +func (cfg *Config) Validate() error { + // TODO: validate those mandatory fields (if exist) in the config + return nil +} diff --git a/processor/awsapmprocessor/config_test.go b/processor/awsapmprocessor/config_test.go new file mode 100644 index 0000000000..ec6bfd223e --- /dev/null +++ b/processor/awsapmprocessor/config_test.go @@ -0,0 +1,104 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsapmprocessor + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap/confmaptest" + + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/customconfiguration" +) + +func TestLoadConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + id component.ID + expected component.Config + errorMessage string + }{ + { + id: component.NewIDWithName("awsapm", ""), + expected: &Config{ + Resolvers: []string{"eks"}, + Rules: []customconfiguration.Rule{ + { + Selectors: []customconfiguration.Selector{ + { + Dimension: "Operation", + Match: "* /api/visits/*", + }, + { + Dimension: "RemoteOperation", + Match: "*", + }, + }, + Action: "keep", + RuleName: "keep01", + }, + { + Selectors: []customconfiguration.Selector{ + { + Dimension: "RemoteService", + Match: "UnknownRemoteService", + }, + { + Dimension: "RemoteOperation", + Match: "GetShardIterator", + }, + }, + Action: "drop", + }, + { + Selectors: []customconfiguration.Selector{ + { + Dimension: "Operation", + Match: "* /api/visits/*", + }, + { + Dimension: "RemoteOperation", + Match: "*", + }, + }, + Replacements: []customconfiguration.Replacement{ + { + TargetDimension: "RemoteOperation", + Value: "ListPetsByCustomer", + }, + { + TargetDimension: "ResourceTarget", + Value: " ", + }, + }, + Action: "replace", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.id.String(), func(t *testing.T) { + cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) + require.NoError(t, err) + + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub(tt.id.String()) + require.NoError(t, err) + require.NoError(t, component.UnmarshalConfig(sub, cfg)) + + if tt.expected == nil { + assert.EqualError(t, component.ValidateConfig(cfg), tt.errorMessage) + return + } + assert.NoError(t, component.ValidateConfig(cfg)) + assert.Equal(t, tt.expected, cfg) + }) + } +} diff --git a/processor/awsapmprocessor/factory.go b/processor/awsapmprocessor/factory.go new file mode 100644 index 0000000000..c91290885d --- /dev/null +++ b/processor/awsapmprocessor/factory.go @@ -0,0 +1,96 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsapmprocessor + +import ( + "context" + "errors" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/processor" + "go.opentelemetry.io/collector/processor/processorhelper" +) + +const ( + // The value of "type" key in configuration. + typeStr = "awsapm" + // The stability level of the processor. + stability = component.StabilityLevelBeta +) + +var consumerCapabilities = consumer.Capabilities{MutatesData: true} + +// NewFactory returns a new factory for the aws attributes processor. +func NewFactory() processor.Factory { + return processor.NewFactory( + typeStr, + createDefaultConfig, + processor.WithTraces(createTracesProcessor, stability), + processor.WithMetrics(createMetricsProcessor, stability), + ) +} + +func createDefaultConfig() component.Config { + return &Config{ + Resolvers: []string{"eks"}, // we need to change the default config later + } +} + +func createTracesProcessor( + ctx context.Context, + set processor.CreateSettings, + cfg component.Config, + next consumer.Traces, +) (processor.Traces, error) { + ap, err := createProcessor(set, cfg) + if err != nil { + return nil, err + } + + return processorhelper.NewTracesProcessor( + ctx, + set, + cfg, + next, + ap.processTraces, + processorhelper.WithCapabilities(consumerCapabilities), + processorhelper.WithStart(ap.Start), + processorhelper.WithShutdown(ap.Shutdown)) +} + +func createMetricsProcessor( + ctx context.Context, + set processor.CreateSettings, + cfg component.Config, + nextMetricsConsumer consumer.Metrics, +) (processor.Metrics, error) { + ap, err := createProcessor(set, cfg) + if err != nil { + return nil, err + } + + return processorhelper.NewMetricsProcessor( + ctx, + set, + cfg, + nextMetricsConsumer, + ap.processMetrics, + processorhelper.WithCapabilities(consumerCapabilities), + processorhelper.WithStart(ap.Start), + processorhelper.WithShutdown(ap.Shutdown)) +} + +func createProcessor( + params processor.CreateSettings, + cfg component.Config, +) (*awsapmprocessor, error) { + pCfg, ok := cfg.(*Config) + if !ok { + return nil, errors.New("could not initialize awsapmprocessor") + } + ap := &awsapmprocessor{logger: params.Logger, config: pCfg} + + return ap, nil +} diff --git a/processor/awsapmprocessor/internal/customconfiguration/common.go b/processor/awsapmprocessor/internal/customconfiguration/common.go new file mode 100644 index 0000000000..3b426f80cc --- /dev/null +++ b/processor/awsapmprocessor/internal/customconfiguration/common.go @@ -0,0 +1,116 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package customconfiguration + +import ( + "fmt" + + "github.com/gobwas/glob" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +type Selector struct { + Dimension string `mapstructure:"dimension"` + Match string `mapstructure:"match"` +} + +type Replacement struct { + TargetDimension string `mapstructure:"target_dimension"` + Value string `mapstructure:"value"` +} + +type Rule struct { + Selectors []Selector `mapstructure:"selectors"` + Replacements []Replacement `mapstructure:"replacements,omitempty"` + Action string `mapstructure:"action"` + RuleName string `mapstructure:"rule_name,omitempty"` +} + +type SelectorMatcherItem struct { + Key string + Matcher glob.Glob +} + +type ActionItem struct { + SelectorMatchers []SelectorMatcherItem + Replacements []Replacement `mapstructure:",omitempty"` +} + +var traceKeyMap = map[string]string{ + "Service": "aws.local.service", + "Operation": "aws.local.operation", + "RemoteService": "aws.remote.service", + "RemoteOperation": "aws.remote.operation", +} + +func getExactKey(metricDimensionKey string, isTrace bool) string { + if !isTrace { + return metricDimensionKey + } + traceDimensionKey, ok := traceKeyMap[metricDimensionKey] + if !ok { + // return original key if there is no matches + return metricDimensionKey + } + return traceDimensionKey +} + +func isSelected(attributes pcommon.Map, selectorMatchers []SelectorMatcherItem, isTrace bool) (bool, error) { + for _, item := range selectorMatchers { + exactKey := getExactKey(item.Key, isTrace) + value, ok := attributes.Get(exactKey) + if !ok { + return false, fmt.Errorf("can not find attribute %q in the datapoint", exactKey) + } + if !item.Matcher.Match(value.AsString()) { + return false, nil + } + } + return true, nil +} + +func generateSelectorMatchers(selectors []Selector) []SelectorMatcherItem { + var selectorMatchers []SelectorMatcherItem + for _, selector := range selectors { + selectorMatcherItem := SelectorMatcherItem{ + selector.Dimension, + glob.MustCompile(selector.Match), + } + selectorMatchers = append(selectorMatchers, selectorMatcherItem) + } + return selectorMatchers +} + +func generateTestAttributes(service string, operation string, remoteService string, remoteOperation string, + isTrace bool) pcommon.Map { + attributes := pcommon.NewMap() + if isTrace { + attributes.PutStr("aws.local.service", service) + attributes.PutStr("aws.local.operation", operation) + attributes.PutStr("aws.remote.service", remoteService) + attributes.PutStr("aws.remote.operation", remoteOperation) + } else { + attributes.PutStr("Service", service) + attributes.PutStr("Operation", operation) + attributes.PutStr("RemoteService", remoteService) + attributes.PutStr("RemoteOperation", remoteOperation) + } + return attributes +} + +func generateActionDetails(rules []Rule, action string) []ActionItem { + var actionItems []ActionItem + for _, rule := range rules { + if rule.Action == action { + var selectorMatchers = generateSelectorMatchers(rule.Selectors) + actionItem := ActionItem{ + selectorMatchers, + rule.Replacements, + } + actionItems = append(actionItems, actionItem) + } + } + + return actionItems +} diff --git a/processor/awsapmprocessor/internal/customconfiguration/dropper.go b/processor/awsapmprocessor/internal/customconfiguration/dropper.go new file mode 100644 index 0000000000..35956c637a --- /dev/null +++ b/processor/awsapmprocessor/internal/customconfiguration/dropper.go @@ -0,0 +1,35 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package customconfiguration + +import "go.opentelemetry.io/collector/pdata/pcommon" + +type DropActions struct { + Actions []ActionItem +} + +func NewCustomDropper(rules []Rule) *DropActions { + return &DropActions{ + Actions: generateActionDetails(rules, "drop"), + } +} + +func (d *DropActions) ShouldBeDropped(attributes, _ pcommon.Map) (bool, error) { + // nothing will be dropped if no rule is defined + if d.Actions == nil || len(d.Actions) == 0 { + return false, nil + } + for _, element := range d.Actions { + isMatched, err := isSelected(attributes, element.SelectorMatchers, false) + if isMatched { + // The datapoint will be dropped if one of keep rule matched + return true, nil + } + if err != nil { + // The datapoint will be kept if error is occurred in match process + return false, err + } + } + return false, nil +} diff --git a/processor/awsapmprocessor/internal/customconfiguration/dropper_test.go b/processor/awsapmprocessor/internal/customconfiguration/dropper_test.go new file mode 100644 index 0000000000..0b1be34859 --- /dev/null +++ b/processor/awsapmprocessor/internal/customconfiguration/dropper_test.go @@ -0,0 +1,185 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package customconfiguration + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +type TestCaseForDropper struct { + name string + input pcommon.Map + output bool +} + +func TestDropperProcessor(t *testing.T) { + config := []Rule{ + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "PUT *", + }, + { + Dimension: "RemoteService", + Match: "customer-test", + }, + }, + Action: "keep", + }, + { + Selectors: []Selector{ + { + Dimension: "RemoteService", + Match: "customer-*", + }, + { + Dimension: "RemoteOperation", + Match: "GET /Owners/*", + }, + }, + Action: "drop", + }, + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "PUT /*/pet/*", + }, + { + Dimension: "RemoteService", + Match: "visit-*-service", + }, + }, + Action: "drop", + }, + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "* /api/visits/*", + }, + { + Dimension: "RemoteOperation", + Match: "*", + }, + }, + Replacements: []Replacement{ + { + TargetDimension: "RemoteOperation", + Value: "ListPetsByCustomer", + }, + { + TargetDimension: "ResourceTarget", + Value: " ", + }, + }, + Action: "replace", + }, + } + + testDropper := NewCustomDropper(config) + testMapPlaceHolder := pcommon.NewMap() + isTrace := false + + testCases := []TestCaseForDropper{ + { + name: "commonTest01ShouldBeKept", + input: generateTestAttributes("customer-test", "GET /user/123", "visit-service", "GET /visit/12345", isTrace), + output: false, + }, + { + name: "commonTest02ShouldBeDropped", + input: generateTestAttributes("common-test", "GET /user/123", "customer-service", "GET /Owners/12345", isTrace), + output: true, + }, + { + name: "commonTest03ShouldBeDropped", + input: generateTestAttributes("common-test", "PUT /test/pet/123", "visit-test-service", "GET /visit/12345", isTrace), + output: true, + }, + } + + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + result, err := testDropper.ShouldBeDropped(tt.input, testMapPlaceHolder) + assert.NoError(t, err) + assert.Equal(t, tt.output, result) + }) + } +} + +func TestDropperProcessorWithNilConfig(t *testing.T) { + + testDropper := NewCustomDropper(nil) + testMapPlaceHolder := pcommon.NewMap() + isTrace := false + + testCases := []TestCaseForDropper{ + { + name: "nilTest01ShouldBeKept", + input: generateTestAttributes("customer-test", "GET /user/123", "visit-service", "GET /visit/12345", isTrace), + output: false, + }, + { + name: "nilTest02ShouldBeDropped", + input: generateTestAttributes("common-test", "GET /user/123", "customer-service", "GET /Owners/12345", isTrace), + output: false, + }, + { + name: "nilTest03ShouldBeDropped", + input: generateTestAttributes("common-test", "PUT /test/pet/123", "visit-test-service", "GET /visit/12345", isTrace), + output: false, + }, + } + + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + result, err := testDropper.ShouldBeDropped(tt.input, testMapPlaceHolder) + assert.NoError(t, err) + assert.Equal(t, tt.output, result) + }) + } +} + +func TestDropperProcessorWithEmptyConfig(t *testing.T) { + + config := []Rule{} + + testDropper := NewCustomDropper(config) + testMapPlaceHolder := pcommon.NewMap() + isTrace := false + + testCases := []TestCaseForDropper{ + { + name: "emptyTest01ShouldBeKept", + input: generateTestAttributes("customer-test", "GET /user/123", "visit-service", "GET /visit/12345", isTrace), + output: false, + }, + { + name: "emptyTest02ShouldBeDropped", + input: generateTestAttributes("common-test", "GET /user/123", "customer-service", "GET /Owners/12345", isTrace), + output: false, + }, + { + name: "emptyTest03ShouldBeDropped", + input: generateTestAttributes("common-test", "PUT /test/pet/123", "visit-test-service", "GET /visit/12345", isTrace), + output: false, + }, + } + + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + result, err := testDropper.ShouldBeDropped(tt.input, testMapPlaceHolder) + assert.NoError(t, err) + assert.Equal(t, tt.output, result) + }) + } +} diff --git a/processor/awsapmprocessor/internal/customconfiguration/keeper.go b/processor/awsapmprocessor/internal/customconfiguration/keeper.go new file mode 100644 index 0000000000..33a8eb9261 --- /dev/null +++ b/processor/awsapmprocessor/internal/customconfiguration/keeper.go @@ -0,0 +1,35 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package customconfiguration + +import "go.opentelemetry.io/collector/pdata/pcommon" + +type KeepActions struct { + Actions []ActionItem +} + +func NewCustomKeeper(rules []Rule) *KeepActions { + return &KeepActions{ + Actions: generateActionDetails(rules, "keep"), + } +} + +func (k *KeepActions) ShouldBeDropped(attributes, _ pcommon.Map) (bool, error) { + // nothing will be dropped if no keep rule is defined + if k.Actions == nil || len(k.Actions) == 0 { + return false, nil + } + for _, element := range k.Actions { + isMatched, err := isSelected(attributes, element.SelectorMatchers, false) + if isMatched { + // The data point will not be dropped if one of keep rule matched + return false, nil + } + if err != nil { + // The data point will be dropped when error is found + return true, err + } + } + return true, nil +} diff --git a/processor/awsapmprocessor/internal/customconfiguration/keeper_test.go b/processor/awsapmprocessor/internal/customconfiguration/keeper_test.go new file mode 100644 index 0000000000..9d1a7a478c --- /dev/null +++ b/processor/awsapmprocessor/internal/customconfiguration/keeper_test.go @@ -0,0 +1,178 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package customconfiguration + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +type TestCaseForKeeper struct { + name string + input pcommon.Map + output bool +} + +func TestKeeperProcessor(t *testing.T) { + config := []Rule{ + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "PUT *", + }, + { + Dimension: "RemoteService", + Match: "customer-test", + }, + }, + Action: "keep", + }, + { + Selectors: []Selector{ + { + Dimension: "RemoteService", + Match: "UnknownRemoteService", + }, + { + Dimension: "RemoteOperation", + Match: "GetShardIterator", + }, + }, + Action: "drop", + }, + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "* /api/visits/*", + }, + { + Dimension: "RemoteOperation", + Match: "*", + }, + }, + Replacements: []Replacement{ + { + TargetDimension: "RemoteOperation", + Value: "ListPetsByCustomer", + }, + { + TargetDimension: "ResourceTarget", + Value: " ", + }, + }, + Action: "replace", + }, + } + + testKeeper := NewCustomKeeper(config) + testMapPlaceHolder := pcommon.NewMap() + isTrace := false + + testCases := []TestCaseForKeeper{ + { + name: "commonTest01ShouldBeKept", + input: generateTestAttributes("visit-test", "PUT owners", "customer-test", "PUT owners", isTrace), + output: false, + }, + { + name: "commonTest02ShouldBeDropped", + input: generateTestAttributes("visit-test", "PUT owners", "vet-test", "PUT owners", isTrace), + output: true, + }, + { + name: "commonTest03ShouldBeDropped", + input: generateTestAttributes("vet-test", "GET owners", "customer-test", "PUT owners", isTrace), + output: true, + }, + } + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + result, err := testKeeper.ShouldBeDropped(tt.input, testMapPlaceHolder) + assert.NoError(t, err) + assert.Equal(t, tt.output, result) + }) + } +} + +func TestKeeperProcessorWithNilConfig(t *testing.T) { + testKeeper := NewCustomKeeper(nil) + testMapPlaceHolder := pcommon.NewMap() + isTrace := false + + testCases := []TestCaseForKeeper{ + { + name: "nilTest01ShouldBeKept", + input: generateTestAttributes("visit-test", "PUT owners", "customer-test", "PUT owners", isTrace), + output: false, + }, + { + name: "nilTest02ShouldBeKept", + input: generateTestAttributes("visit-test", "PUT owners", "vet-test", "PUT owners", isTrace), + output: false, + }, + { + name: "nilTest03ShouldBeKept", + input: generateTestAttributes("vet-test", "PUT owners", "visit-test", "PUT owners", isTrace), + output: false, + }, + { + name: "nilTest04ShouldBeKept", + input: generateTestAttributes("customer-test", "PUT owners", "visit-test", "PUT owners", isTrace), + output: false, + }, + } + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + result, err := testKeeper.ShouldBeDropped(tt.input, testMapPlaceHolder) + assert.NoError(t, err) + assert.Equal(t, tt.output, result) + }) + } +} + +func TestKeeperProcessorWithEmptyConfig(t *testing.T) { + + config := []Rule{} + + testKeeper := NewCustomKeeper(config) + testMapPlaceHolder := pcommon.NewMap() + isTrace := false + + testCases := []TestCaseForKeeper{ + { + name: "emptyTest01ShouldBeKept", + input: generateTestAttributes("visit-test", "PUT owners", "customer-test", "PUT owners", isTrace), + output: false, + }, + { + name: "emptyTest02ShouldBeKept", + input: generateTestAttributes("visit-test", "PUT owners", "vet-test", "PUT owners", isTrace), + output: false, + }, + { + name: "emptyTest03ShouldBeKept", + input: generateTestAttributes("vet-test", "PUT owners", "visit-test", "PUT owners", isTrace), + output: false, + }, + { + name: "emptyTest04ShouldBeKept", + input: generateTestAttributes("customer-test", "PUT owners", "visit-test", "PUT owners", isTrace), + output: false, + }, + } + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + result, err := testKeeper.ShouldBeDropped(tt.input, testMapPlaceHolder) + assert.NoError(t, err) + assert.Equal(t, tt.output, result) + }) + } +} diff --git a/processor/awsapmprocessor/internal/customconfiguration/replacer.go b/processor/awsapmprocessor/internal/customconfiguration/replacer.go new file mode 100644 index 0000000000..34555bb214 --- /dev/null +++ b/processor/awsapmprocessor/internal/customconfiguration/replacer.go @@ -0,0 +1,54 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package customconfiguration + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" +) + +type ReplaceActions struct { + Actions []ActionItem +} + +func NewCustomReplacer(rules []Rule) *ReplaceActions { + return &ReplaceActions{ + generateActionDetails(rules, "replace"), + } +} + +func (r *ReplaceActions) Process(attributes, _ pcommon.Map, isTrace bool) error { + // do nothing when there is no replace rule defined + if r.Actions == nil || len(r.Actions) == 0 { + return nil + } + // If there are more than one rule are matched, the last one will be executed(Later one has higher priority) + actions := r.Actions + finalRules := make(map[string]string) + for i := len(actions) - 1; i >= 0; i = i - 1 { + element := actions[i] + isMatched, _ := isSelected(attributes, element.SelectorMatchers, isTrace) + if !isMatched { + continue + } + for _, replacement := range element.Replacements { + targetDimensionKey := getExactKey(replacement.TargetDimension, isTrace) + // don't allow customer add new dimension key + _, isExist := attributes.Get(targetDimensionKey) + if !isExist { + continue + } + // every replacement in one specific dimension only will be performed once + _, ok := finalRules[targetDimensionKey] + if ok { + continue + } + finalRules[targetDimensionKey] = replacement.Value + } + } + + for key, value := range finalRules { + attributes.PutStr(key, value) + } + return nil +} diff --git a/processor/awsapmprocessor/internal/customconfiguration/replacer_test.go b/processor/awsapmprocessor/internal/customconfiguration/replacer_test.go new file mode 100644 index 0000000000..2d1e9ec751 --- /dev/null +++ b/processor/awsapmprocessor/internal/customconfiguration/replacer_test.go @@ -0,0 +1,281 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package customconfiguration + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +type TestCaseForReplacer struct { + name string + input pcommon.Map + output pcommon.Map + isTrace bool +} + +func TestReplacerProcess(t *testing.T) { + + config := []Rule{ + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "PUT *", + }, + { + Dimension: "RemoteService", + Match: "customer-test", + }, + }, + Action: "keep", + }, + { + Selectors: []Selector{ + { + Dimension: "RemoteService", + Match: "UnknownRemoteService", + }, + { + Dimension: "RemoteOperation", + Match: "GetShardIterator", + }, + }, + Action: "drop", + }, + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "* /api/visits/*", + }, + { + Dimension: "RemoteOperation", + Match: "*", + }, + }, + Replacements: []Replacement{ + { + TargetDimension: "RemoteOperation", + Value: "ListPetsByCustomer", + }, + { + TargetDimension: "Operation", + Value: "PUT/GET", + }, + }, + Action: "replace", + }, + } + + testReplacer := NewCustomReplacer(config) + testMapPlaceHolder := pcommon.NewMap() + + testCases := []TestCaseForReplacer{ + { + name: "test01TraceMatch", + input: generateTestAttributes("replace-test", "PUT /api/visits/test/123456", "customer-test", + "GET", true), + output: generateTestAttributes("replace-test", "PUT/GET", "customer-test", + "ListPetsByCustomer", true), + isTrace: true, + }, + { + name: "test02TraceNotMatch", + input: generateTestAttributes("replace-test", "PUT /api/customer/owners/12345", "customer-test", + "GET", true), + output: generateTestAttributes("replace-test", "PUT /api/customer/owners/12345", "customer-test", + "GET", true), + isTrace: true, + }, + { + name: "test03MetricMatch", + input: generateTestAttributes("replace-test", "PUT /api/visits/owners/12345", "customer-test", + "GET", false), + output: generateTestAttributes("replace-test", "PUT/GET", "customer-test", + "ListPetsByCustomer", false), + isTrace: false, + }, + { + name: "test04MetricNotMatch", + input: generateTestAttributes("replace-test", "PUT /api/customer/owners/12345", "customer-test", + "GET", false), + output: generateTestAttributes("replace-test", "PUT /api/customer/owners/12345", "customer-test", + "GET", false), + isTrace: false, + }, + } + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + assert.NoError(t, testReplacer.Process(tt.input, testMapPlaceHolder, tt.isTrace)) + assert.Equal(t, tt.output, tt.input) + }) + } +} + +func TestReplacerProcessWithPriority(t *testing.T) { + + config := []Rule{ + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "* /api/visits/*", + }, + { + Dimension: "RemoteOperation", + Match: "*", + }, + }, + Replacements: []Replacement{ + { + TargetDimension: "RemoteOperation", + Value: "ListPetsByCustomer", + }, + { + TargetDimension: "Operation", + Value: "PUT/GET", + }, + }, + Action: "replace", + }, + { + Selectors: []Selector{ + { + Dimension: "Operation", + Match: "PUT /api/visits/*", + }, + { + Dimension: "RemoteOperation", + Match: "PUT *", + }, + }, + Replacements: []Replacement{ + { + TargetDimension: "RemoteOperation", + Value: "PUT visits", + }, + { + TargetDimension: "Operation", + Value: "PUT", + }, + }, + Action: "replace", + }, + } + + testReplacer := NewCustomReplacer(config) + testMapPlaceHolder := pcommon.NewMap() + + testCases := []TestCaseForReplacer{ + { + name: "test01TraceMatchPreviousOne", + input: generateTestAttributes("replace-test", "PUT /api/visits/test/123456", "customer-test", + "GET", true), + output: generateTestAttributes("replace-test", "PUT/GET", "customer-test", + "ListPetsByCustomer", true), + isTrace: true, + }, + { + name: "test02TraceBothMatch", + input: generateTestAttributes("replace-test", "PUT /api/visits/test/123456", "customer-test", + "PUT /api/owners/123456", true), + output: generateTestAttributes("replace-test", "PUT", "customer-test", + "PUT visits", true), + isTrace: true, + }, + { + name: "test03MetricMatchPreviousOne", + input: generateTestAttributes("replace-test", "PUT /api/visits/owners/12345", "customer-test", + "GET", false), + output: generateTestAttributes("replace-test", "PUT/GET", "customer-test", + "ListPetsByCustomer", false), + isTrace: false, + }, + { + name: "test04MetricBothMatch", + input: generateTestAttributes("replace-test", "PUT /api/visits/owners/12345", "customer-test", + "PUT owners", false), + output: generateTestAttributes("replace-test", "PUT", "customer-test", + "PUT visits", false), + isTrace: false, + }, + } + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + assert.NoError(t, testReplacer.Process(tt.input, testMapPlaceHolder, tt.isTrace)) + assert.Equal(t, tt.output, tt.input) + }) + } +} + +func TestReplacerProcessWithNilConfig(t *testing.T) { + + testReplacer := NewCustomReplacer(nil) + testMapPlaceHolder := pcommon.NewMap() + + testCases := []TestCaseForReplacer{ + { + name: "test01Trace", + input: generateTestAttributes("replace-test", "PUT /api/visits/test/123456", "customer-test", + "GET", true), + output: generateTestAttributes("replace-test", "PUT /api/visits/test/123456", "customer-test", + "GET", true), + isTrace: true, + }, + { + name: "test02Metric", + input: generateTestAttributes("replace-test", "PUT /api/visits/owners/12345", "customer-test", + "GET", false), + output: generateTestAttributes("replace-test", "PUT /api/visits/owners/12345", "customer-test", + "GET", false), + isTrace: false, + }, + } + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + assert.NoError(t, testReplacer.Process(tt.input, testMapPlaceHolder, tt.isTrace)) + assert.Equal(t, tt.output, tt.input) + }) + } +} + +func TestReplacerProcessWithEmptyConfig(t *testing.T) { + + config := []Rule{} + + testReplacer := NewCustomReplacer(config) + testMapPlaceHolder := pcommon.NewMap() + + testCases := []TestCaseForReplacer{ + { + name: "test01Trace", + input: generateTestAttributes("replace-test", "PUT /api/visits/test/123456", "customer-test", + "GET", true), + output: generateTestAttributes("replace-test", "PUT /api/visits/test/123456", "customer-test", + "GET", true), + isTrace: true, + }, + { + name: "test02Metric", + input: generateTestAttributes("replace-test", "PUT /api/visits/owners/12345", "customer-test", + "GET", false), + output: generateTestAttributes("replace-test", "PUT /api/visits/owners/12345", "customer-test", + "GET", false), + isTrace: false, + }, + } + for i := range testCases { + tt := testCases[i] + t.Run(tt.name, func(t *testing.T) { + assert.NoError(t, testReplacer.Process(tt.input, testMapPlaceHolder, tt.isTrace)) + assert.Equal(t, tt.output, tt.input) + }) + } +} diff --git a/processor/awsapmprocessor/internal/normalizer/attributesnormalizer.go b/processor/awsapmprocessor/internal/normalizer/attributesnormalizer.go new file mode 100644 index 0000000000..a72e73f01c --- /dev/null +++ b/processor/awsapmprocessor/internal/normalizer/attributesnormalizer.go @@ -0,0 +1,107 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package normalizer + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + "go.uber.org/zap" +) + +type attributesNormalizer struct { + logger *zap.Logger +} + +var renameMapForMetric = map[string]string{ + "aws.local.service": "Service", + "aws.local.operation": "Operation", + "aws.remote.service": "RemoteService", + "aws.remote.operation": "RemoteOperation", + "aws.remote.target": "RemoteTarget", +} + +var renameMapForTrace = map[string]string{ + // these kubernetes resource attributes are set by the openTelemtry operator + // see the code referecnes from upstream: + // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 + // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 + "k8s.deployment.name": "K8s.Workload", + "k8s.statefulset.name": "K8s.Workload", + "k8s.daemonset.name": "K8s.Workload", + "k8s.job.name": "K8s.Workload", + "k8s.cronjob.name": "K8s.Workload", + "k8s.pod.name": "K8s.Pod", +} + +var copyMapForMetric = map[string]string{ + // these kubernetes resource attributes are set by the openTelemtry operator + // see the code referecnes from upstream: + // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 + // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 + "k8s.deployment.name": "K8s.Workload", + "k8s.statefulset.name": "K8s.Workload", + "k8s.daemonset.name": "K8s.Workload", + "k8s.job.name": "K8s.Workload", + "k8s.cronjob.name": "K8s.Workload", + "k8s.pod.name": "K8s.Pod", +} + +func NewAttributesNormalizer(logger *zap.Logger) *attributesNormalizer { + return &attributesNormalizer{ + logger: logger, + } +} + +func (n *attributesNormalizer) Process(attributes, resourceAttributes pcommon.Map, isTrace bool) error { + n.copyResourceAttributesToAttributes(attributes, resourceAttributes, isTrace) + n.renameAttributes(attributes, resourceAttributes, isTrace) + return nil +} + +func (n *attributesNormalizer) renameAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { + attrs := attributes + renameMap := renameMapForMetric + if isTrace { + attrs = resourceAttributes + renameMap = renameMapForTrace + } + + rename(attrs, renameMap) +} + +func (n *attributesNormalizer) copyResourceAttributesToAttributes(attributes, resourceAttributes pcommon.Map, isTrace bool) { + if isTrace { + return + } + for k, v := range copyMapForMetric { + if resourceAttrValue, ok := resourceAttributes.Get(k); ok { + // print some debug info when an attribute value is overwritten + if originalAttrValue, ok := attributes.Get(k); ok { + n.logger.Debug("attribute value is overwritten", zap.String("attribute", k), zap.String("original", originalAttrValue.AsString()), zap.String("new", resourceAttrValue.AsString())) + } + attributes.PutStr(v, resourceAttrValue.AsString()) + if k == "k8s.pod.name" { + // only copy "host.id" from resource attributes to "K8s.Node" in attributesif the pod name is set + if host, ok := resourceAttributes.Get("host.id"); ok { + attributes.PutStr("K8s.Node", host.AsString()) + } + } + } + } +} + +func rename(attrs pcommon.Map, renameMap map[string]string) { + for original, replacement := range renameMap { + if value, ok := attrs.Get(original); ok { + attrs.PutStr(replacement, value.AsString()) + attrs.Remove(original) + if original == "k8s.pod.name" { + // only rename host.id if the pod name is set + if host, ok := attrs.Get("host.id"); ok { + attrs.PutStr("K8s.Node", host.AsString()) + attrs.Remove("host.id") + } + } + } + } +} diff --git a/processor/awsapmprocessor/internal/normalizer/attributesnormalizer_test.go b/processor/awsapmprocessor/internal/normalizer/attributesnormalizer_test.go new file mode 100644 index 0000000000..4ca77b3f61 --- /dev/null +++ b/processor/awsapmprocessor/internal/normalizer/attributesnormalizer_test.go @@ -0,0 +1,105 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package normalizer + +import ( + "testing" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.uber.org/zap" +) + +func TestRenameAttributes_for_metric(t *testing.T) { + logger, _ := zap.NewDevelopment() + normalizer := NewAttributesNormalizer(logger) + + // test for metric + // Create a pcommon.Map with some attributes + attributes := pcommon.NewMap() + for originalKey, replacementKey := range renameMapForMetric { + attributes.PutStr(originalKey, replacementKey+"-value") + } + + resourceAttributes := pcommon.NewMap() + // Call the process method + normalizer.renameAttributes(attributes, resourceAttributes, false) + + // Check that the original key has been removed + for originalKey := range renameMapForMetric { + if _, ok := attributes.Get(originalKey); ok { + t.Errorf("originalKey was not removed") + } + } + + // Check that the new key has the correct value + for _, replacementKey := range renameMapForMetric { + if value, ok := attributes.Get(replacementKey); !ok || value.AsString() != replacementKey+"-value" { + t.Errorf("replacementKey has incorrect value: got %v, want %v", value.AsString(), replacementKey+"-value") + } + } +} + +func TestRenameAttributes_for_trace(t *testing.T) { + logger, _ := zap.NewDevelopment() + normalizer := NewAttributesNormalizer(logger) + + // test for trace + // Create a pcommon.Map with some attributes + resourceAttributes := pcommon.NewMap() + for originalKey, replacementKey := range renameMapForTrace { + resourceAttributes.PutStr(originalKey, replacementKey+"-value") + } + resourceAttributes.PutStr("host.id", "i-01ef7d37f42caa168") + + attributes := pcommon.NewMap() + // Call the process method + normalizer.renameAttributes(attributes, resourceAttributes, true) + + // Check that the original key has been removed + for originalKey := range renameMapForTrace { + if _, ok := resourceAttributes.Get(originalKey); ok { + t.Errorf("originalKey was not removed") + } + } + + // Check that the new key has the correct value + for _, replacementKey := range renameMapForTrace { + if value, ok := resourceAttributes.Get(replacementKey); !ok || value.AsString() != replacementKey+"-value" { + t.Errorf("replacementKey has incorrect value: got %v, want %v", value.AsString(), replacementKey+"-value") + } + } + + if value, ok := resourceAttributes.Get("K8s.Node"); !ok || value.AsString() != "i-01ef7d37f42caa168" { + t.Errorf("replacementKey has incorrect value: got %v, want %v", value.AsString(), "i-01ef7d37f42caa168") + } +} + +func TestCopyResourceAttributesToAttributes(t *testing.T) { + logger, _ := zap.NewDevelopment() + normalizer := NewAttributesNormalizer(logger) + + // Create a pcommon.Map for resourceAttributes with some attributes + resourceAttributes := pcommon.NewMap() + for resourceAttrKey, attrKey := range copyMapForMetric { + resourceAttributes.PutStr(resourceAttrKey, attrKey+"-value") + } + resourceAttributes.PutStr("host.id", "i-01ef7d37f42caa168") + + // Create a pcommon.Map for attributes + attributes := pcommon.NewMap() + + // Call the process method + normalizer.copyResourceAttributesToAttributes(attributes, resourceAttributes, false) + + // Check that the attribute has been copied correctly + for _, attrKey := range copyMapForMetric { + if value, ok := attributes.Get(attrKey); !ok || value.AsString() != attrKey+"-value" { + t.Errorf("Attribute was not copied correctly: got %v, want %v", value.AsString(), attrKey+"-value") + } + } + + if value, ok := attributes.Get("K8s.Node"); !ok || value.AsString() != "i-01ef7d37f42caa168" { + t.Errorf("Attribute was not copied correctly: got %v, want %v", value.AsString(), "i-01ef7d37f42caa168") + } +} diff --git a/processor/awsapmprocessor/internal/resolver/attributesresolver.go b/processor/awsapmprocessor/internal/resolver/attributesresolver.go new file mode 100644 index 0000000000..c7f243daa9 --- /dev/null +++ b/processor/awsapmprocessor/internal/resolver/attributesresolver.go @@ -0,0 +1,52 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resolver + +import ( + "context" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.uber.org/zap" +) + +type subResolver interface { + Process(attributes, resourceAttributes pcommon.Map) error + Stop(ctx context.Context) error +} + +type attributesResolver struct { + subResolvers []subResolver +} + +// create a new attributes resolver +func NewAttributesResolver(resolverNames []string, logger *zap.Logger) *attributesResolver { + subResolvers := []subResolver{} + for _, resolverName := range resolverNames { + if resolverName == "eks" { + subResolvers = append(subResolvers, getEksResolver(logger)) + } + } + return &attributesResolver{ + subResolvers: subResolvers, + } +} + +// Process the attributes +func (r *attributesResolver) Process(attributes, resourceAttributes pcommon.Map, _ bool) error { + for _, subResolver := range r.subResolvers { + if err := subResolver.Process(attributes, resourceAttributes); err != nil { + return err + } + } + return nil +} + +func (r *attributesResolver) Stop(ctx context.Context) error { + for _, subResolver := range r.subResolvers { + if err := subResolver.Stop(ctx); err != nil { + return err + } + } + return nil +} diff --git a/processor/awsapmprocessor/internal/resolver/attributesresolver_test.go b/processor/awsapmprocessor/internal/resolver/attributesresolver_test.go new file mode 100644 index 0000000000..2597c4926f --- /dev/null +++ b/processor/awsapmprocessor/internal/resolver/attributesresolver_test.go @@ -0,0 +1,67 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resolver + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +type MockSubResolver struct { + mock.Mock +} + +func (m *MockSubResolver) Process(attributes, resourceAttributes pcommon.Map) error { + args := m.Called(attributes, resourceAttributes) + return args.Error(0) +} + +func (m *MockSubResolver) Stop(ctx context.Context) error { + args := m.Called(ctx) + return args.Error(0) +} + +func TestAttributesResolver_Process(t *testing.T) { + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + + mockSubResolver1 := new(MockSubResolver) + mockSubResolver1.On("Process", attributes, resourceAttributes).Return(nil) + + mockSubResolver2 := new(MockSubResolver) + mockSubResolver2.On("Process", attributes, resourceAttributes).Return(errors.New("error")) + + r := &attributesResolver{ + subResolvers: []subResolver{mockSubResolver1, mockSubResolver2}, + } + + err := r.Process(attributes, resourceAttributes, true) + assert.Error(t, err) + mockSubResolver1.AssertExpectations(t) + mockSubResolver2.AssertExpectations(t) +} + +func TestAttributesResolver_Stop(t *testing.T) { + ctx := context.Background() + + mockSubResolver1 := new(MockSubResolver) + mockSubResolver1.On("Stop", ctx).Return(nil) + + mockSubResolver2 := new(MockSubResolver) + mockSubResolver2.On("Stop", ctx).Return(errors.New("error")) + + r := &attributesResolver{ + subResolvers: []subResolver{mockSubResolver1, mockSubResolver2}, + } + + err := r.Stop(ctx) + assert.Error(t, err) + mockSubResolver1.AssertExpectations(t) + mockSubResolver2.AssertExpectations(t) +} diff --git a/processor/awsapmprocessor/internal/resolver/eks.go b/processor/awsapmprocessor/internal/resolver/eks.go new file mode 100644 index 0000000000..ee5ae21880 --- /dev/null +++ b/processor/awsapmprocessor/internal/resolver/eks.go @@ -0,0 +1,709 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resolver + +import ( + "context" + "errors" + "fmt" + "math/rand" + "net" + "regexp" + "strconv" + "strings" + "sync" + "time" + + mapset "github.com/deckarep/golang-set/v2" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + // kubeAllowedStringAlphaNums holds the characters allowed in replicaset names from as parent deployment + // https://github.com/kubernetes/apimachinery/blob/master/pkg/util/rand/rand.go#L83 + kubeAllowedStringAlphaNums = "bcdfghjklmnpqrstvwxz2456789" + + // Deletion delay adjustment: + // Previously, EKS resolver would instantly remove the IP to Service mapping when a pod was destroyed. + // This posed a problem because: + // 1. Metric data is aggregated and emitted every 1 minute. + // 2. If this aggregated metric data, which contains the IP of the now-destroyed pod, arrives + // at the EKS resolver after the IP records have already been deleted, the metric can't be processed correctly. + // + // To mitigate this issue, we've introduced a 2-minute deletion delay. This ensures that any + // metric data that arrives within those 2 minutes, containing the old IP, will still get mapped correctly to a service. + deletionDelay = 2 * time.Minute +) + +var ( + // ReplicaSet name = Deployment name + "-" + 10 alphanumeric characters long string (see https://stackoverflow.com/questions/46204504/kubernetes-pod-naming-convention and https://stackoverflow.com/questions/71090356/how-does-random-string-in-kubernetes-pod-name-decided) + // the alphanumeric characters in Kubernetes are restricted to exclude vowels to reduce the chances of "normal words" being formed. (see https://github.com/kubernetes/apimachinery/blob/master/pkg/util/rand/rand.go#L83) + replicaSetWithDeploymentNamePattern = fmt.Sprintf(`^(.+)-[%s]{10}$`, kubeAllowedStringAlphaNums) + deploymentFromReplicaSetPattern = regexp.MustCompile(replicaSetWithDeploymentNamePattern) + // if a pod is launched directly by a replicaSet (with a given name by users), its name has the following pattern: + // Pod name = ReplicaSet name + 5 alphanumeric characters long string + podWithReplicaSetNamePattern = fmt.Sprintf(`^(.+)-[%s]{5}$`, kubeAllowedStringAlphaNums) + replicaSetFromPodPattern = regexp.MustCompile(podWithReplicaSetNamePattern) +) + +type eksResolver struct { + clusterName string // used to save the cluster name as a cache + logger *zap.Logger + clientset kubernetes.Interface + ipToPod *sync.Map + podToWorkloadAndNamespace *sync.Map + ipToServiceAndNamespace *sync.Map + serviceAndNamespaceToSelectors *sync.Map + workloadAndNamespaceToLabels *sync.Map + serviceToWorkload *sync.Map // computed from serviceAndNamespaceToSelectors and workloadAndNamespaceToLabels every 1 min + workloadPodCount map[string]int + safeStopCh *safeChannel // trace and metric processors share the same eksResolver and might close the same channel separately +} + +// a safe channel which can be closed multiple times +type safeChannel struct { + sync.Mutex + + ch chan struct{} + closed bool +} + +func (sc *safeChannel) Close() { + sc.Lock() + defer sc.Unlock() + + if !sc.closed { + close(sc.ch) + sc.closed = true + } +} + +var ( + once sync.Once + instance *eksResolver +) + +func jitterSleep(seconds int) { + jitter := time.Duration(rand.Intn(seconds)) * time.Second // nolint:gosec + time.Sleep(jitter) +} + +func attachNamespace(resourceName, namespace string) string { + // character "@" is not allowed in kubernetes resource names: https://unofficial-kubernetes.readthedocs.io/en/latest/concepts/overview/working-with-objects/names/ + return resourceName + "@" + namespace +} + +func getServiceAndNamespace(service *corev1.Service) string { + return attachNamespace(service.Name, service.Namespace) +} + +func extractResourceAndNamespace(serviceOrWorkloadAndNamespace string) (string, string) { + // extract service name and namespace from serviceAndNamespace + parts := strings.Split(serviceOrWorkloadAndNamespace, "@") + if len(parts) != 2 { + return "", "" + } + return parts[0], parts[1] +} + +func extractWorkloadNameFromRS(replicaSetName string) (string, error) { + match := deploymentFromReplicaSetPattern.FindStringSubmatch(replicaSetName) + if match != nil { + return match[1], nil + } + + return "", errors.New("failed to extract workload name from replicatSet name: " + replicaSetName) +} + +func extractWorkloadNameFromPodName(podName string) (string, error) { + match := replicaSetFromPodPattern.FindStringSubmatch(podName) + if match != nil { + return match[1], nil + } + + return "", errors.New("failed to extract workload name from pod name: " + podName) +} + +func getWorkloadAndNamespace(pod *corev1.Pod) string { + var workloadAndNamespace string + if pod.ObjectMeta.OwnerReferences != nil { + for _, ownerRef := range pod.ObjectMeta.OwnerReferences { + if workloadAndNamespace != "" { + break + } + + if ownerRef.Kind == "ReplicaSet" { + if workloadName, err := extractWorkloadNameFromRS(ownerRef.Name); err == nil { + // when the replicaSet is created by a deployment, use deployment name + workloadAndNamespace = attachNamespace(workloadName, pod.Namespace) + } else if workloadName, err := extractWorkloadNameFromPodName(pod.Name); err == nil { + // when the replicaSet is not created by a deployment, use replicaSet name directly + workloadAndNamespace = attachNamespace(workloadName, pod.Namespace) + } + } else if ownerRef.Kind == "StatefulSet" { + workloadAndNamespace = attachNamespace(ownerRef.Name, pod.Namespace) + } else if ownerRef.Kind == "DaemonSet" { + workloadAndNamespace = attachNamespace(ownerRef.Name, pod.Namespace) + } + } + } + + return workloadAndNamespace +} + +// Deleter represents a type that can delete a key from a map after a certain delay. +type Deleter interface { + DeleteWithDelay(m *sync.Map, key interface{}) +} + +// TimedDeleter deletes a key after a specified delay. +type TimedDeleter struct { + Delay time.Duration +} + +func (td *TimedDeleter) DeleteWithDelay(m *sync.Map, key interface{}) { + go func() { + time.Sleep(td.Delay) + m.Delete(key) + }() +} + +func onAddOrUpdateService(obj interface{}, ipToServiceAndNamespace, serviceAndNamespaceToSelectors *sync.Map) { + service := obj.(*corev1.Service) + // service can also have an external IP (or ingress IP) that could be accessed + // this field can be either an IP address (in some edge case) or a hostname (see "EXTERNAL-IP" column in "k get svc" output) + // [ec2-user@ip-172-31-11-104 one-step]$ k get svc -A + // NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + // default pet-clinic-frontend ClusterIP 10.100.216.182 8080/TCP 108m + // default vets-service ClusterIP 10.100.62.167 8083/TCP 108m + // default visits-service ClusterIP 10.100.96.5 8082/TCP 108m + // ingress-nginx default-http-backend ClusterIP 10.100.11.231 80/TCP 108m + // ingress-nginx ingress-nginx LoadBalancer 10.100.154.5 aex7997ece08c435dbd2b912fd5aa5bd-5372117830.xxxxx.elb.amazonaws.com 80:32080/TCP,443:32081/TCP,9113:30410/TCP 108m + // kube-system kube-dns ClusterIP 10.100.0.10 + // + // we ignore such case for now and may need to consider it in the future + if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != "None" { + ipToServiceAndNamespace.Store(service.Spec.ClusterIP, getServiceAndNamespace(service)) + } + labelSet := mapset.NewSet[string]() + for key, value := range service.Spec.Selector { + labelSet.Add(key + "=" + value) + } + if labelSet.Cardinality() > 0 { + serviceAndNamespaceToSelectors.Store(getServiceAndNamespace(service), labelSet) + } +} + +func onDeleteService(obj interface{}, ipToServiceAndNamespace, serviceAndNamespaceToSelectors *sync.Map, deleter Deleter) { + service := obj.(*corev1.Service) + if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != "None" { + deleter.DeleteWithDelay(ipToServiceAndNamespace, service.Spec.ClusterIP) + } + deleter.DeleteWithDelay(serviceAndNamespaceToSelectors, getServiceAndNamespace(service)) +} + +func removeHostNetworkRecords(pod *corev1.Pod, ipToPod *sync.Map, deleter Deleter) { + for _, port := range getHostNetworkPorts(pod) { + deleter.DeleteWithDelay(ipToPod, pod.Status.HostIP+":"+port) + } +} + +func updateHostNetworkRecords(newPod *corev1.Pod, oldPod *corev1.Pod, ipToPod *sync.Map, deleter Deleter) { + newHostIPPorts := make(map[string]bool) + oldHostIPPorts := make(map[string]bool) + + for _, port := range getHostNetworkPorts(newPod) { + newHostIPPorts[newPod.Status.HostIP+":"+port] = true + } + + for _, port := range getHostNetworkPorts(oldPod) { + oldHostIPPorts[oldPod.Status.HostIP+":"+port] = true + } + + for oldHostIPPort := range oldHostIPPorts { + if _, exist := newHostIPPorts[oldHostIPPort]; !exist { + deleter.DeleteWithDelay(ipToPod, oldHostIPPort) + } + } + + for newHostIPPort := range newHostIPPorts { + if _, exist := oldHostIPPorts[newHostIPPort]; !exist { + ipToPod.Store(newHostIPPort, newPod.Name) + } + } +} + +func handlePodAdd(pod *corev1.Pod, ipToPod *sync.Map) { + if pod.Spec.HostNetwork { + for _, port := range getHostNetworkPorts(pod) { + ipToPod.Store(pod.Status.HostIP+":"+port, pod.Name) + } + } else if pod.Status.PodIP != "" { + ipToPod.Store(pod.Status.PodIP, pod.Name) + } +} + +func handlePodUpdate(newPod *corev1.Pod, oldPod *corev1.Pod, ipToPod *sync.Map, deleter Deleter) { + if oldPod.Spec.HostNetwork && newPod.Spec.HostNetwork { + // Case 1: Both oldPod and newPod are using host network + // Here we need to update the host network records accordingly + updateHostNetworkRecords(newPod, oldPod, ipToPod, deleter) + } else if oldPod.Spec.HostNetwork && !newPod.Spec.HostNetwork { + // Case 2: The oldPod was using the host network, but the newPod is not + // Here we remove the old host network records and add new PodIP record if it is not empty + removeHostNetworkRecords(oldPod, ipToPod, deleter) + if newPod.Status.PodIP != "" { + ipToPod.Store(newPod.Status.PodIP, newPod.Name) + } + } else if !oldPod.Spec.HostNetwork && newPod.Spec.HostNetwork { + // Case 3: The oldPod was not using the host network, but the newPod is + // Here we remove the old PodIP record and add new host network records + if oldPod.Status.PodIP != "" { + deleter.DeleteWithDelay(ipToPod, oldPod.Status.PodIP) + } + for _, port := range getHostNetworkPorts(newPod) { + ipToPod.Store(newPod.Status.HostIP+":"+port, newPod.Name) + } + } else if !oldPod.Spec.HostNetwork && !newPod.Spec.HostNetwork && oldPod.Status.PodIP != newPod.Status.PodIP { + // Case 4: Both oldPod and newPod are not using the host network, but the Pod IPs are different + // Here we replace the old PodIP record with the new one + if oldPod.Status.PodIP != "" { + deleter.DeleteWithDelay(ipToPod, oldPod.Status.PodIP) + } + if newPod.Status.PodIP != "" { + ipToPod.Store(newPod.Status.PodIP, newPod.Name) + } + } +} + +func onAddOrUpdatePod(newObj, oldObj interface{}, ipToPod, podToWorkloadAndNamespace, workloadAndNamespaceToLabels *sync.Map, workloadPodCount map[string]int, isAdd bool, logger *zap.Logger, deleter Deleter) { + pod := newObj.(*corev1.Pod) + + if isAdd { + handlePodAdd(pod, ipToPod) + } else { + oldPod := oldObj.(*corev1.Pod) + handlePodUpdate(pod, oldPod, ipToPod, deleter) + } + + workloadAndNamespace := getWorkloadAndNamespace(pod) + + if workloadAndNamespace != "" { + podToWorkloadAndNamespace.Store(pod.Name, workloadAndNamespace) + podLabels := mapset.NewSet[string]() + for key, value := range pod.ObjectMeta.Labels { + podLabels.Add(key + "=" + value) + } + if podLabels.Cardinality() > 0 { + workloadAndNamespaceToLabels.Store(workloadAndNamespace, podLabels) + } + if isAdd { + workloadPodCount[workloadAndNamespace]++ + logger.Debug("Added pod", zap.String("pod", pod.Name), zap.String("workload", workloadAndNamespace), zap.Int("count", workloadPodCount[workloadAndNamespace])) + } + } +} + +func onDeletePod(obj interface{}, ipToPod, podToWorkloadAndNamespace, workloadAndNamespaceToLabels *sync.Map, workloadPodCount map[string]int, logger *zap.Logger, deleter Deleter) { + pod := obj.(*corev1.Pod) + if pod.Status.PodIP != "" { + deleter.DeleteWithDelay(ipToPod, pod.Status.PodIP) + } else if pod.Status.HostIP != "" { + for _, port := range getHostNetworkPorts(pod) { + deleter.DeleteWithDelay(ipToPod, pod.Status.HostIP+":"+port) + } + } + + if workloadKey, ok := podToWorkloadAndNamespace.Load(pod.Name); ok { + workloadAndNamespace := workloadKey.(string) + workloadPodCount[workloadAndNamespace]-- + logger.Debug("workload pod count", zap.String("workload", workloadAndNamespace), zap.Int("podCount", workloadPodCount[workloadAndNamespace])) + if workloadPodCount[workloadAndNamespace] == 0 { + deleter.DeleteWithDelay(workloadAndNamespaceToLabels, workloadAndNamespace) + } + } + deleter.DeleteWithDelay(podToWorkloadAndNamespace, pod.Name) +} + +type PodWatcher struct { + ipToPod *sync.Map + podToWorkloadAndNamespace *sync.Map + workloadAndNamespaceToLabels *sync.Map + workloadPodCount map[string]int + logger *zap.Logger + informer cache.SharedIndexInformer + deleter Deleter +} + +func NewPodWatcher(logger *zap.Logger, informer cache.SharedIndexInformer, deleter Deleter) *PodWatcher { + return &PodWatcher{ + ipToPod: &sync.Map{}, + podToWorkloadAndNamespace: &sync.Map{}, + workloadAndNamespaceToLabels: &sync.Map{}, + workloadPodCount: make(map[string]int), + logger: logger, + informer: informer, + deleter: deleter, + } +} + +func (p *PodWatcher) Run(stopCh chan struct{}) { + p.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + p.logger.Debug("list and watch for pods: ADD") + onAddOrUpdatePod(obj, nil, p.ipToPod, p.podToWorkloadAndNamespace, p.workloadAndNamespaceToLabels, p.workloadPodCount, true, p.logger, p.deleter) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + p.logger.Debug("list and watch for pods: UPDATE") + onAddOrUpdatePod(newObj, oldObj, p.ipToPod, p.podToWorkloadAndNamespace, p.workloadAndNamespaceToLabels, p.workloadPodCount, false, p.logger, p.deleter) + }, + DeleteFunc: func(obj interface{}) { + p.logger.Debug("list and watch for pods: DELETE") + onDeletePod(obj, p.ipToPod, p.podToWorkloadAndNamespace, p.workloadAndNamespaceToLabels, p.workloadPodCount, p.logger, p.deleter) + }, + }) + + go p.informer.Run(stopCh) + +} + +func (p *PodWatcher) WaitForCacheSync(stopCh chan struct{}) { + if !cache.WaitForNamedCacheSync("podWatcher", stopCh, p.informer.HasSynced) { + p.logger.Fatal("timed out waiting for kubernetes pod watcher caches to sync") + } + + p.logger.Info("PodWatcher: Cache synced") +} + +type ServiceWatcher struct { + ipToServiceAndNamespace *sync.Map + serviceAndNamespaceToSelectors *sync.Map + logger *zap.Logger + informer cache.SharedIndexInformer + deleter Deleter +} + +func NewServiceWatcher(logger *zap.Logger, informer cache.SharedIndexInformer, deleter Deleter) *ServiceWatcher { + return &ServiceWatcher{ + ipToServiceAndNamespace: &sync.Map{}, + serviceAndNamespaceToSelectors: &sync.Map{}, + logger: logger, + informer: informer, + deleter: deleter, + } +} + +func (s *ServiceWatcher) Run(stopCh chan struct{}) { + s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + s.logger.Debug("list and watch for services: ADD") + onAddOrUpdateService(obj, s.ipToServiceAndNamespace, s.serviceAndNamespaceToSelectors) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + s.logger.Debug("list and watch for services: UPDATE") + onAddOrUpdateService(newObj, s.ipToServiceAndNamespace, s.serviceAndNamespaceToSelectors) + }, + DeleteFunc: func(obj interface{}) { + s.logger.Debug("list and watch for services: DELETE") + onDeleteService(obj, s.ipToServiceAndNamespace, s.serviceAndNamespaceToSelectors, s.deleter) + }, + }) + go s.informer.Run(stopCh) +} + +func (s *ServiceWatcher) WaitForCacheSync(stopCh chan struct{}) { + if !cache.WaitForNamedCacheSync("serviceWatcher", stopCh, s.informer.HasSynced) { + s.logger.Fatal("timed out waiting for kubernetes service watcher caches to sync") + } + + s.logger.Info("ServiceWatcher: Cache synced") +} + +type ServiceToWorkloadMapper struct { + serviceAndNamespaceToSelectors *sync.Map + workloadAndNamespaceToLabels *sync.Map + serviceToWorkload *sync.Map + logger *zap.Logger + deleter Deleter +} + +func NewServiceToWorkloadMapper(serviceAndNamespaceToSelectors, workloadAndNamespaceToLabels, serviceToWorkload *sync.Map, logger *zap.Logger, deleter Deleter) *ServiceToWorkloadMapper { + return &ServiceToWorkloadMapper{ + serviceAndNamespaceToSelectors: serviceAndNamespaceToSelectors, + workloadAndNamespaceToLabels: workloadAndNamespaceToLabels, + serviceToWorkload: serviceToWorkload, + logger: logger, + deleter: deleter, + } +} + +func (m *ServiceToWorkloadMapper) MapServiceToWorkload() { + m.logger.Debug("Map service to workload at:", zap.Time("time", time.Now())) + + m.serviceAndNamespaceToSelectors.Range(func(key, value interface{}) bool { + var workloads []string + serviceAndNamespace := key.(string) + _, serviceNamespace := extractResourceAndNamespace(serviceAndNamespace) + serviceLabels := value.(mapset.Set[string]) + + m.workloadAndNamespaceToLabels.Range(func(workloadKey, labelsValue interface{}) bool { + labels := labelsValue.(mapset.Set[string]) + workloadAndNamespace := workloadKey.(string) + _, workloadNamespace := extractResourceAndNamespace(workloadAndNamespace) + if workloadNamespace == serviceNamespace && workloadNamespace != "" && serviceLabels.IsSubset(labels) { + m.logger.Debug("Found workload for service", zap.String("service", serviceAndNamespace), zap.String("workload", workloadAndNamespace)) + workloads = append(workloads, workloadAndNamespace) + } + + return true + }) + + if len(workloads) > 1 { + m.logger.Info("Multiple workloads found for service. You will get unexpected results.", zap.String("service", serviceAndNamespace), zap.Strings("workloads", workloads)) + } else if len(workloads) == 1 { + m.serviceToWorkload.Store(serviceAndNamespace, workloads[0]) + } else { + m.logger.Debug("No workload found for service", zap.String("service", serviceAndNamespace)) + m.deleter.DeleteWithDelay(m.serviceToWorkload, serviceAndNamespace) + } + return true + }) +} + +func (m *ServiceToWorkloadMapper) Start(stopCh chan struct{}) { + // do the first mapping immediately + m.MapServiceToWorkload() + m.logger.Debug("First-time map service to workload at:", zap.Time("time", time.Now())) + + go func() { + for { + select { + case <-stopCh: + return + case <-time.After(time.Minute + 30*time.Second): + m.MapServiceToWorkload() + m.logger.Debug("Map service to workload at:", zap.Time("time", time.Now())) + } + } + }() +} + +func getEksResolver(logger *zap.Logger) subResolver { + once.Do(func() { + config, err := clientcmd.BuildConfigFromFlags("", "") + if err != nil { + logger.Fatal("Failed to create config", zap.Error(err)) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + logger.Fatal("Failed to create eks client", zap.Error(err)) + } + + // add a time jitter of 10 seconds + jitterSleep(10) + + sharedInformerFactory := informers.NewSharedInformerFactory(clientset, 0) + podInformer := sharedInformerFactory.Core().V1().Pods().Informer() + serviceInformer := sharedInformerFactory.Core().V1().Services().Informer() + + timedDeleter := &TimedDeleter{Delay: deletionDelay} + podWatcher := NewPodWatcher(logger, podInformer, timedDeleter) + serviceWatcher := NewServiceWatcher(logger, serviceInformer, timedDeleter) + + safeStopCh := &safeChannel{ch: make(chan struct{}), closed: false} + // initialize the pod and service watchers for the cluster + podWatcher.Run(safeStopCh.ch) + serviceWatcher.Run(safeStopCh.ch) + // wait for caches to sync (for once) so that clients knows about the pods and services in the cluster + podWatcher.WaitForCacheSync(safeStopCh.ch) + serviceWatcher.WaitForCacheSync(safeStopCh.ch) + + serviceToWorkload := &sync.Map{} + serviceToWorkloadMapper := NewServiceToWorkloadMapper(serviceWatcher.serviceAndNamespaceToSelectors, podWatcher.workloadAndNamespaceToLabels, serviceToWorkload, logger, timedDeleter) + serviceToWorkloadMapper.Start(safeStopCh.ch) + + instance = &eksResolver{ + logger: logger, + clientset: clientset, + ipToServiceAndNamespace: serviceWatcher.ipToServiceAndNamespace, + serviceAndNamespaceToSelectors: serviceWatcher.serviceAndNamespaceToSelectors, + ipToPod: podWatcher.ipToPod, + podToWorkloadAndNamespace: podWatcher.podToWorkloadAndNamespace, + workloadAndNamespaceToLabels: podWatcher.workloadAndNamespaceToLabels, + serviceToWorkload: serviceToWorkload, + workloadPodCount: podWatcher.workloadPodCount, + safeStopCh: safeStopCh, + } + + go instance.debugPrint() + }) + + return instance +} + +func (e *eksResolver) Stop(_ context.Context) error { + e.safeStopCh.Close() + return nil +} + +// add a method to eksResolver +func (e *eksResolver) GetWorkloadAndNamespaceByIP(ip string) (string, string, error) { + var workload, namespace string + if podKey, ok := e.ipToPod.Load(ip); ok { + pod := podKey.(string) + if workloadKey, ok := e.podToWorkloadAndNamespace.Load(pod); ok { + workload, namespace = extractResourceAndNamespace(workloadKey.(string)) + return workload, namespace, nil + } + } + + if serviceKey, ok := e.ipToServiceAndNamespace.Load(ip); ok { + serviceAndNamespace := serviceKey.(string) + if workloadKey, ok := e.serviceToWorkload.Load(serviceAndNamespace); ok { + workload, namespace = extractResourceAndNamespace(workloadKey.(string)) + return workload, namespace, nil + } + } + + return "", "", errors.New("no EKS workload found for ip: " + ip) +} + +func printSyncMap(name string, m *sync.Map, logger *zap.Logger) { + logger.Debug("", zap.String("MapName", name)) + m.Range(func(key, value interface{}) bool { + logger.Debug("", zap.Any("key", key), zap.Any("value", value)) + return true + }) + logger.Debug("DEBUG ====================") +} + +func (e *eksResolver) debugPrint() { + // call some logic every 5 minutes for ever + for { + select { + case <-time.After(5 * time.Minute): + e.debug() + case <-e.safeStopCh.ch: + return + } + } +} + +func (e *eksResolver) debug() { + e.logger.Debug("start debug print") + // print ipToServiceAndNamespace + printSyncMap("ipToServiceAndNamespace", e.ipToServiceAndNamespace, e.logger) + + // print serviceAndNamespaceToSelectors + printSyncMap("serviceAndNamespaceToSelectors", e.serviceAndNamespaceToSelectors, e.logger) + + // print ipToPod + printSyncMap("ipToPod", e.ipToPod, e.logger) + + // print podToWorkloadAndNamespace + printSyncMap("podToWorkloadAndNamespace", e.podToWorkloadAndNamespace, e.logger) + + // print workloadAndNamespaceToLabels + printSyncMap("workloadAndNamespaceToLabels", e.workloadAndNamespaceToLabels, e.logger) + + // print serviceToWorkload + e.logger.Debug("workload pod count", zap.Any("workloadPodCount", e.workloadPodCount)) + printSyncMap("serviceToWorkload", e.serviceToWorkload, e.logger) + e.logger.Debug("end debug print") +} + +func (e *eksResolver) Process(attributes, resourceAttributes pcommon.Map) error { + if value, ok := attributes.Get("aws.remote.service"); ok { + valueStr := value.AsString() + ipStr := "" + if ip, _, ok := extractIPPort(valueStr); ok { + if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(valueStr); err == nil { + attributes.PutStr("aws.remote.service", workload) + attributes.PutStr("K8s.RemoteNamespace", namespace) + } else { + ipStr = ip + } + } else if isIP(valueStr) { + ipStr = valueStr + } + + if ipStr != "" { + if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(ipStr); err == nil { + attributes.PutStr("aws.remote.service", workload) + attributes.PutStr("K8s.RemoteNamespace", namespace) + } else { + e.logger.Debug("failed to Process ip", zap.String("ip", ipStr), zap.Error(err)) + attributes.PutStr("aws.remote.service", "UnknownRemoteService") + } + } + } + + if namespace, ok := resourceAttributes.Get("k8s.namespace.name"); ok { + attributes.PutStr("K8s.Namespace", namespace.AsString()) + // assuming cluster name resource attribute like "ec2.tag.kubernetes.io/cluster/petclinic-test" always exist when "k8s.namespace.name" exist + if e.clusterName != "" { + attributes.PutStr("EKS.Cluster", e.clusterName) + } else { + // iterate resource attributes to find the cluster name + resourceAttributes.Range(func(key string, value pcommon.Value) bool { + if strings.HasPrefix(key, "ec2.tag.kubernetes.io/cluster/") && value.Type() == pcommon.ValueTypeStr && value.AsString() == "owned" { + e.clusterName = strings.TrimPrefix(key, "ec2.tag.kubernetes.io/cluster/") + attributes.PutStr("EKS.Cluster", e.clusterName) + return false + } + return true + }) + } + } + return nil +} + +func isIP(ipString string) bool { + ip := net.ParseIP(ipString) + return ip != nil +} + +const IP_PORT_PATTERN = `^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)$` + +var ipPortRegex = regexp.MustCompile(IP_PORT_PATTERN) + +func extractIPPort(ipPort string) (string, string, bool) { + match := ipPortRegex.MatchString(ipPort) + + if !match { + return "", "", false + } + + result := ipPortRegex.FindStringSubmatch(ipPort) + if len(result) != 3 { + return "", "", false + } + + ip := result[1] + port := result[2] + + return ip, port, true +} + +func getHostNetworkPorts(pod *corev1.Pod) []string { + var ports []string + if !pod.Spec.HostNetwork { + return ports + } + for _, container := range pod.Spec.Containers { + for _, port := range container.Ports { + if port.HostPort != 0 { + ports = append(ports, strconv.Itoa(int(port.HostPort))) + } + } + } + return ports +} diff --git a/processor/awsapmprocessor/internal/resolver/eks_test.go b/processor/awsapmprocessor/internal/resolver/eks_test.go new file mode 100644 index 0000000000..984672fc89 --- /dev/null +++ b/processor/awsapmprocessor/internal/resolver/eks_test.go @@ -0,0 +1,1114 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resolver + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + mapset "github.com/deckarep/golang-set/v2" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// MockDeleter deletes a key immediately, useful for testing. +type MockDeleter struct{} + +func (md *MockDeleter) DeleteWithDelay(m *sync.Map, key interface{}) { + m.Delete(key) +} + +var mockDeleter = &MockDeleter{} + +// TestAttachNamespace function +func TestAttachNamespace(t *testing.T) { + result := attachNamespace("testResource", "testNamespace") + if result != "testResource@testNamespace" { + t.Errorf("attachNamespace was incorrect, got: %s, want: %s.", result, "testResource@testNamespace") + } +} + +// TestGetServiceAndNamespace function +func TestGetServiceAndNamespace(t *testing.T) { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testService", + Namespace: "testNamespace", + }, + } + result := getServiceAndNamespace(service) + if result != "testService@testNamespace" { + t.Errorf("getServiceAndNamespace was incorrect, got: %s, want: %s.", result, "testService@testNamespace") + } +} + +// TestExtractResourceAndNamespace function +func TestExtractResourceAndNamespace(t *testing.T) { + // Test normal case + name, namespace := extractResourceAndNamespace("testService@testNamespace") + if name != "testService" || namespace != "testNamespace" { + t.Errorf("extractResourceAndNamespace was incorrect, got: %s and %s, want: %s and %s.", name, namespace, "testService", "testNamespace") + } + + // Test invalid case + name, namespace = extractResourceAndNamespace("invalid") + if name != "" || namespace != "" { + t.Errorf("extractResourceAndNamespace was incorrect, got: %s and %s, want: %s and %s.", name, namespace, "", "") + } +} + +func TestExtractWorkloadNameFromRS(t *testing.T) { + testCases := []struct { + name string + replicaSetName string + want string + shouldErr bool + }{ + { + name: "Valid ReplicaSet Name", + replicaSetName: "my-deployment-5859ffc7ff", + want: "my-deployment", + shouldErr: false, + }, + { + name: "Invalid ReplicaSet Name - No Hyphen", + replicaSetName: "mydeployment5859ffc7ff", + want: "", + shouldErr: true, + }, + { + name: "Invalid ReplicaSet Name - Less Than 10 Suffix Characters", + replicaSetName: "my-deployment-bc2", + want: "", + shouldErr: true, + }, + { + name: "Invalid ReplicaSet Name - More Than 10 Suffix Characters", + replicaSetName: "my-deployment-5859ffc7ffx", + want: "", + shouldErr: true, + }, + { + name: "Invalid ReplicaSet Name - Invalid Characters in Suffix", + replicaSetName: "my-deployment-aeiou12345", + want: "", + shouldErr: true, + }, + { + name: "Invalid ReplicaSet Name - Empty String", + replicaSetName: "", + want: "", + shouldErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := extractWorkloadNameFromRS(tc.replicaSetName) + + if (err != nil) != tc.shouldErr { + t.Errorf("extractWorkloadNameFromRS() error = %v, wantErr %v", err, tc.shouldErr) + return + } + + if got != tc.want { + t.Errorf("extractWorkloadNameFromRS() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestExtractWorkloadNameFromPodName(t *testing.T) { + testCases := []struct { + name string + podName string + want string + shouldErr bool + }{ + { + name: "Valid Pod Name", + podName: "my-replicaset-bc24f", + want: "my-replicaset", + shouldErr: false, + }, + { + name: "Invalid Pod Name - No Hyphen", + podName: "myreplicasetbc24f", + want: "", + shouldErr: true, + }, + { + name: "Invalid Pod Name - Less Than 5 Suffix Characters", + podName: "my-replicaset-bc2", + want: "", + shouldErr: true, + }, + { + name: "Invalid Pod Name - More Than 5 Suffix Characters", + podName: "my-replicaset-bc24f5", + want: "", + shouldErr: true, + }, + { + name: "Invalid Pod Name - Empty String", + podName: "", + want: "", + shouldErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := extractWorkloadNameFromPodName(tc.podName) + + if (err != nil) != tc.shouldErr { + t.Errorf("extractWorkloadNameFromPodName() error = %v, wantErr %v", err, tc.shouldErr) + return + } + + if got != tc.want { + t.Errorf("extractWorkloadNameFromPodName() = %v, want %v", got, tc.want) + } + }) + } +} + +// TestGetWorkloadAndNamespace function +func TestGetWorkloadAndNamespace(t *testing.T) { + // Test ReplicaSet case + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPod", + Namespace: "testNamespace", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testDeployment-5d68bc5f49", + }, + }, + }, + } + result := getWorkloadAndNamespace(pod) + if result != "testDeployment@testNamespace" { + t.Errorf("getDeploymentAndNamespace was incorrect, got: %s, want: %s.", result, "testDeployment@testNamespace") + } + + // Test StatefulSet case + pod.ObjectMeta.OwnerReferences[0].Kind = "StatefulSet" + pod.ObjectMeta.OwnerReferences[0].Name = "testStatefulSet" + result = getWorkloadAndNamespace(pod) + if result != "testStatefulSet@testNamespace" { + t.Errorf("getWorkloadAndNamespace was incorrect, got: %s, want: %s.", result, "testStatefulSet@testNamespace") + } + + // Test Other case + pod.ObjectMeta.OwnerReferences[0].Kind = "Other" + pod.ObjectMeta.OwnerReferences[0].Name = "testOther" + result = getWorkloadAndNamespace(pod) + if result != "" { + t.Errorf("getWorkloadAndNamespace was incorrect, got: %s, want: %s.", result, "") + } + + // Test no OwnerReferences case + pod.ObjectMeta.OwnerReferences = nil + result = getWorkloadAndNamespace(pod) + if result != "" { + t.Errorf("getWorkloadAndNamespace was incorrect, got: %s, want: %s.", result, "") + } +} + +func TestServiceToWorkloadMapper_MapServiceToWorkload(t *testing.T) { + logger, _ := zap.NewDevelopment() + + serviceAndNamespaceToSelectors := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + serviceToWorkload := &sync.Map{} + + serviceAndNamespaceToSelectors.Store("service1@namespace1", mapset.NewSet("label1=value1", "label2=value2")) + workloadAndNamespaceToLabels.Store("deployment1@namespace1", mapset.NewSet("label1=value1", "label2=value2", "label3=value3")) + + mapper := NewServiceToWorkloadMapper(serviceAndNamespaceToSelectors, workloadAndNamespaceToLabels, serviceToWorkload, logger, mockDeleter) + mapper.MapServiceToWorkload() + + if _, ok := serviceToWorkload.Load("service1@namespace1"); !ok { + t.Errorf("Expected service1@namespace1 to be mapped to a workload, but it was not") + } +} + +func TestServiceToWorkloadMapper_MapServiceToWorkload_NoWorkload(t *testing.T) { + logger, _ := zap.NewDevelopment() + + serviceAndNamespaceToSelectors := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + serviceToWorkload := &sync.Map{} + + // Add a service with no matching workload + serviceAndNamespace := "service@namespace" + serviceAndNamespaceToSelectors.Store(serviceAndNamespace, mapset.NewSet("label1=value1")) + serviceToWorkload.Store(serviceAndNamespace, "workload@namespace") + + mapper := NewServiceToWorkloadMapper(serviceAndNamespaceToSelectors, workloadAndNamespaceToLabels, serviceToWorkload, logger, mockDeleter) + mapper.MapServiceToWorkload() + + // Check that the service was deleted from serviceToWorkload + if _, ok := serviceToWorkload.Load(serviceAndNamespace); ok { + t.Errorf("Service was not deleted from serviceToWorkload") + } +} + +func TestServiceToWorkloadMapper_MapServiceToWorkload_MultipleWorkloads(t *testing.T) { + logger, _ := zap.NewDevelopment() + + serviceAndNamespaceToSelectors := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + serviceToWorkload := &sync.Map{} + + serviceAndNamespace := "service@namespace" + serviceAndNamespaceToSelectors.Store(serviceAndNamespace, mapset.NewSet("label1=value1", "label2=value2")) + + // Add two workloads with matching labels to the service + workloadAndNamespaceToLabels.Store("workload1@namespace", mapset.NewSet("label1=value1", "label2=value2", "label3=value3")) + workloadAndNamespaceToLabels.Store("workload2@namespace", mapset.NewSet("label1=value1", "label2=value2", "label4=value4")) + + mapper := NewServiceToWorkloadMapper(serviceAndNamespaceToSelectors, workloadAndNamespaceToLabels, serviceToWorkload, logger, mockDeleter) + mapper.MapServiceToWorkload() + + // Check that the service does not map to any workload + if _, ok := serviceToWorkload.Load(serviceAndNamespace); ok { + t.Errorf("Unexpected mapping of service to multiple workloads") + } +} + +func TestMapServiceToWorkload_StopsWhenSignaled(t *testing.T) { + logger, _ := zap.NewDevelopment() + + serviceAndNamespaceToSelectors := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + serviceToWorkload := &sync.Map{} + + stopchan := make(chan struct{}) + + // Signal the stopchan to stop after 100 milliseconds + time.AfterFunc(100*time.Millisecond, func() { + close(stopchan) + }) + + mapper := NewServiceToWorkloadMapper(serviceAndNamespaceToSelectors, workloadAndNamespaceToLabels, serviceToWorkload, logger, mockDeleter) + + start := time.Now() + mapper.Start(stopchan) + duration := time.Since(start) + + // Check that the function stopped in a reasonable time after the stop signal + if duration > 200*time.Millisecond { + t.Errorf("mapServiceToWorkload did not stop in a reasonable time after the stop signal, duration: %v", duration) + } +} + +func TestOnAddOrUpdateService(t *testing.T) { + // Create a fake service + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myservice", + Namespace: "mynamespace", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Selector: map[string]string{ + "app": "myapp", + }, + }, + } + + // Create the maps + ipToServiceAndNamespace := &sync.Map{} + serviceAndNamespaceToSelectors := &sync.Map{} + + // Call the function + onAddOrUpdateService(service, ipToServiceAndNamespace, serviceAndNamespaceToSelectors) + + // Check that the maps contain the expected entries + if _, ok := ipToServiceAndNamespace.Load("1.2.3.4"); !ok { + t.Errorf("ipToServiceAndNamespace does not contain the service IP") + } + if _, ok := serviceAndNamespaceToSelectors.Load("myservice@mynamespace"); !ok { + t.Errorf("serviceAndNamespaceToSelectors does not contain the service") + } +} + +func TestOnDeleteService(t *testing.T) { + // Create a fake service + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myservice", + Namespace: "mynamespace", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Selector: map[string]string{ + "app": "myapp", + }, + }, + } + + // Create the maps and add the service to them + ipToServiceAndNamespace := &sync.Map{} + ipToServiceAndNamespace.Store("1.2.3.4", "myservice@mynamespace") + serviceAndNamespaceToSelectors := &sync.Map{} + serviceAndNamespaceToSelectors.Store("myservice@mynamespace", mapset.NewSet("app=myapp")) + + // Call the function + onDeleteService(service, ipToServiceAndNamespace, serviceAndNamespaceToSelectors, mockDeleter) + + // Check that the maps do not contain the service + if _, ok := ipToServiceAndNamespace.Load("1.2.3.4"); ok { + t.Errorf("ipToServiceAndNamespace still contains the service IP") + } + if _, ok := serviceAndNamespaceToSelectors.Load("myservice@mynamespace"); ok { + t.Errorf("serviceAndNamespaceToSelectors still contains the service") + } +} + +func TestOnAddOrUpdatePod(t *testing.T) { + logger, _ := zap.NewProduction() + + t.Run("pod with both PodIP and HostIP", func(t *testing.T) { + ipToPod := &sync.Map{} + podToWorkloadAndNamespace := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + workloadPodCount := map[string]int{} + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPod", + Namespace: "testNamespace", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testDeployment-598b89cd8d", + }, + }, + }, + Status: corev1.PodStatus{ + PodIP: "1.2.3.4", + HostIP: "5.6.7.8", + }, + } + + onAddOrUpdatePod(pod, nil, ipToPod, podToWorkloadAndNamespace, workloadAndNamespaceToLabels, workloadPodCount, true, logger, mockDeleter) + + // Test the mappings in ipToPod + if podName, _ := ipToPod.Load("1.2.3.4"); podName != "testPod" { + t.Errorf("ipToPod was incorrect, got: %s, want: %s.", podName, "testPod") + } + + // Test the mapping in podToWorkloadAndNamespace + if depAndNamespace, _ := podToWorkloadAndNamespace.Load("testPod"); depAndNamespace != "testDeployment@testNamespace" { + t.Errorf("podToWorkloadAndNamespace was incorrect, got: %s, want: %s.", depAndNamespace, "testDeployment@testNamespace") + } + + // Test the count in workloadPodCount + if count := workloadPodCount["testDeployment@testNamespace"]; count != 1 { + t.Errorf("workloadPodCount was incorrect, got: %d, want: %d.", count, 1) + } + }) + + t.Run("pod with only HostIP", func(t *testing.T) { + ipToPod := &sync.Map{} + podToWorkloadAndNamespace := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + workloadPodCount := map[string]int{} + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPod", + Namespace: "testNamespace", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testDeployment-7b74958fb8", + }, + }, + }, + Status: corev1.PodStatus{ + HostIP: "5.6.7.8", + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + { + HostPort: int32(8080), + }, + }, + }, + }, + }, + } + + onAddOrUpdatePod(pod, nil, ipToPod, podToWorkloadAndNamespace, workloadAndNamespaceToLabels, workloadPodCount, true, logger, mockDeleter) + + // Test the mappings in ipToPod + if podName, _ := ipToPod.Load("5.6.7.8:8080"); podName != "testPod" { + t.Errorf("ipToPod was incorrect, got: %s, want: %s.", podName, "testPod") + } + + // Test the mapping in podToWorkloadAndNamespace + if depAndNamespace, _ := podToWorkloadAndNamespace.Load("testPod"); depAndNamespace != "testDeployment@testNamespace" { + t.Errorf("podToWorkloadAndNamespace was incorrect, got: %s, want: %s.", depAndNamespace, "testDeployment@testNamespace") + } + + // Test the count in workloadPodCount + if count := workloadPodCount["testDeployment@testNamespace"]; count != 1 { + t.Errorf("workloadPodCount was incorrect, got: %d, want: %d.", count, 1) + } + }) + + t.Run("pod updated with different set of labels", func(t *testing.T) { + ipToPod := &sync.Map{} + podToWorkloadAndNamespace := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + workloadPodCount := map[string]int{} + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPod", + Namespace: "testNamespace", + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testDeployment-5d68bc5f49", + }, + }, + }, + Status: corev1.PodStatus{ + HostIP: "5.6.7.8", + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + {HostPort: 8080}, + }, + }, + }, + }, + } + + // add the pod + onAddOrUpdatePod(pod, nil, ipToPod, podToWorkloadAndNamespace, workloadAndNamespaceToLabels, workloadPodCount, true, logger, mockDeleter) + + // Test the mappings in ipToPod + if podName, ok := ipToPod.Load("5.6.7.8:8080"); !ok && podName != "testPod" { + t.Errorf("ipToPod[%s] was incorrect, got: %s, want: %s.", "5.6.7.8:8080", podName, "testPod") + } + + // Test the mapping in workloadAndNamespaceToLabels + labels, _ := workloadAndNamespaceToLabels.Load("testDeployment@testNamespace") + expectedLabels := []string{"label1=value1", "label2=value2"} + for _, label := range expectedLabels { + if !labels.(mapset.Set[string]).Contains(label) { + t.Errorf("deploymentAndNamespaceToLabels was incorrect, got: %v, want: %s.", labels, label) + } + } + + pod2 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPod", + Namespace: "testNamespace", + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + "label3": "value3", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testDeployment-5d68bc5f49", + }, + }, + }, + Status: corev1.PodStatus{ + PodIP: "1.2.3.4", + HostIP: "5.6.7.8", + }, + } + + // add the pod + onAddOrUpdatePod(pod2, pod, ipToPod, podToWorkloadAndNamespace, workloadAndNamespaceToLabels, workloadPodCount, false, logger, mockDeleter) + + // Test the mappings in ipToPod + if _, ok := ipToPod.Load("5.6.7.8:8080"); ok { + t.Errorf("ipToPod[%s] should be deleted", "5.6.7.8:8080") + } + + if podName, ok := ipToPod.Load("1.2.3.4"); !ok && podName != "testPod" { + t.Errorf("ipToPod[%s] was incorrect, got: %s, want: %s.", "1.2.3.4", podName, "testPod") + } + // Test the mapping in workloadAndNamespaceToLabels + labels, _ = workloadAndNamespaceToLabels.Load("testDeployment@testNamespace") + expectedLabels = []string{"label1=value1", "label2=value2", "label3=value3"} + for _, label := range expectedLabels { + if !labels.(mapset.Set[string]).Contains(label) { + t.Errorf("workloadAndNamespaceToLabels was incorrect, got: %v, want: %s.", labels, label) + } + } + }) +} + +func TestOnDeletePod(t *testing.T) { + logger, _ := zap.NewProduction() + + t.Run("pod with both PodIP and HostIP", func(t *testing.T) { + ipToPod := &sync.Map{} + podToWorkloadAndNamespace := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + workloadPodCount := map[string]int{} + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPod", + Namespace: "testNamespace", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testDeployment-xyz", + }, + }, + }, + Status: corev1.PodStatus{ + PodIP: "1.2.3.4", + HostIP: "5.6.7.8", + }, + } + + // Assume the pod has already been added + ipToPod.Store(pod.Status.PodIP, pod.Name) + ipToPod.Store(pod.Status.HostIP, pod.Name) + podToWorkloadAndNamespace.Store(pod.Name, "testDeployment@testNamespace") + workloadAndNamespaceToLabels.Store("testDeployment@testNamespace", "testLabels") + workloadPodCount["testDeployment@testNamespace"] = 1 + + onDeletePod(pod, ipToPod, podToWorkloadAndNamespace, workloadAndNamespaceToLabels, workloadPodCount, logger, mockDeleter) + + // Test if the entries in ipToPod and podToWorkloadAndNamespace have been deleted + if _, ok := ipToPod.Load("1.2.3.4"); ok { + t.Errorf("ipToPod deletion was incorrect, key: %s still exists", "1.2.3.4") + } + + if _, ok := podToWorkloadAndNamespace.Load("testPod"); ok { + t.Errorf("podToWorkloadAndNamespace deletion was incorrect, key: %s still exists", "testPod") + } + + // Test if the count in workloadPodCount has been decremented and the entry in workloadAndNamespaceToLabels has been deleted + if count := workloadPodCount["testDeployment@testNamespace"]; count != 0 { + t.Errorf("workloadPodCount was incorrect, got: %d, want: %d.", count, 0) + } + + if _, ok := workloadAndNamespaceToLabels.Load("testDeployment@testNamespace"); ok { + t.Errorf("workloadAndNamespaceToLabels deletion was incorrect, key: %s still exists", "testDeployment@testNamespace") + } + }) + + t.Run("pod with only HostIP and some network ports", func(t *testing.T) { + ipToPod := &sync.Map{} + podToWorkloadAndNamespace := &sync.Map{} + workloadAndNamespaceToLabels := &sync.Map{} + workloadPodCount := map[string]int{} + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPod", + Namespace: "testNamespace", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "testDeployment-xyz", + }, + }, + }, + Status: corev1.PodStatus{ + HostIP: "5.6.7.8", + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + { + HostPort: int32(8080), + }, + }, + }, + }, + }, + } + + // Assume the pod has already been added + ipToPod.Store(pod.Status.HostIP, pod.Name) + ipToPod.Store(pod.Status.HostIP+":8080", pod.Name) + podToWorkloadAndNamespace.Store(pod.Name, "testDeployment@testNamespace") + workloadAndNamespaceToLabels.Store("testDeployment@testNamespace", "testLabels") + workloadPodCount["testDeployment@testNamespace"] = 1 + + onDeletePod(pod, ipToPod, podToWorkloadAndNamespace, workloadAndNamespaceToLabels, workloadPodCount, logger, mockDeleter) + + // Test if the entries in ipToPod and podToWorkloadAndNamespace have been deleted + if _, ok := ipToPod.Load("5.6.7.8:8080"); ok { + t.Errorf("ipToPod deletion was incorrect, key: %s still exists", "5.6.7.8:8080") + } + + if _, ok := podToWorkloadAndNamespace.Load("testPod"); ok { + t.Errorf("podToDeploymentAndNamespace deletion was incorrect, key: %s still exists", "testPod") + } + + // Test if the count in workloadPodCount has been decremented and the entry in workloadAndNamespaceToLabels has been deleted + if count := workloadPodCount["testDeployment@testNamespace"]; count != 0 { + t.Errorf("workloadPodCount was incorrect, got: %d, want: %d.", count, 0) + } + + if _, ok := workloadAndNamespaceToLabels.Load("testDeployment@testNamespace"); ok { + t.Errorf("workloadAndNamespaceToLabels deletion was incorrect, key: %s still exists", "testDeployment@testNamespace") + } + }) +} + +func TestEksResolver(t *testing.T) { + logger, _ := zap.NewProduction() + ctx := context.Background() + + t.Run("Test GetWorkloadAndNamespaceByIP", func(t *testing.T) { + resolver := &eksResolver{ + logger: logger, + ipToPod: &sync.Map{}, + podToWorkloadAndNamespace: &sync.Map{}, + ipToServiceAndNamespace: &sync.Map{}, + serviceToWorkload: &sync.Map{}, + } + + ip := "1.2.3.4" + pod := "testPod" + workloadAndNamespace := "testDeployment@testNamespace" + + // Pre-fill the resolver maps + resolver.ipToPod.Store(ip, pod) + resolver.podToWorkloadAndNamespace.Store(pod, workloadAndNamespace) + + // Test existing IP + workload, namespace, err := resolver.GetWorkloadAndNamespaceByIP(ip) + if err != nil || workload != "testDeployment" || namespace != "testNamespace" { + t.Errorf("Expected testDeployment@testNamespace, got %s@%s, error: %v", workload, namespace, err) + } + + // Test non-existing IP + _, _, err = resolver.GetWorkloadAndNamespaceByIP("5.6.7.8") + if err == nil || !strings.Contains(err.Error(), "no EKS workload found for ip: 5.6.7.8") { + t.Errorf("Expected error, got %v", err) + } + + // Test ip in ipToServiceAndNamespace but not in ipToPod + newIP := "2.3.4.5" + serviceAndNamespace := "testService@testNamespace" + resolver.ipToServiceAndNamespace.Store(newIP, serviceAndNamespace) + resolver.serviceToWorkload.Store(serviceAndNamespace, workloadAndNamespace) + workload, namespace, err = resolver.GetWorkloadAndNamespaceByIP(newIP) + if err != nil || workload != "testDeployment" || namespace != "testNamespace" { + t.Errorf("Expected testDeployment@testNamespace, got %s@%s, error: %v", workload, namespace, err) + } + }) + + t.Run("Test Stop", func(t *testing.T) { + resolver := &eksResolver{ + logger: logger, + safeStopCh: &safeChannel{ch: make(chan struct{}), closed: false}, + } + + err := resolver.Stop(ctx) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if !resolver.safeStopCh.closed { + t.Errorf("Expected channel to be closed") + } + + // Test closing again + err = resolver.Stop(ctx) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + }) + + t.Run("Test Process", func(t *testing.T) { + // helper function to get string values from the attributes + getStrAttr := func(attributes pcommon.Map, key string, t *testing.T) string { + if value, ok := attributes.Get(key); ok { + return value.AsString() + } else { + t.Errorf("Failed to get value for key: %s", key) + return "" + } + } + + logger, _ := zap.NewProduction() + resolver := &eksResolver{ + logger: logger, + ipToPod: &sync.Map{}, + podToWorkloadAndNamespace: &sync.Map{}, + ipToServiceAndNamespace: &sync.Map{}, + serviceToWorkload: &sync.Map{}, + } + + // Test case 1: "aws.remote.service" contains IP:Port + attributes := pcommon.NewMap() + attributes.PutStr("aws.remote.service", "192.0.2.1:8080") + resourceAttributes := pcommon.NewMap() + resolver.ipToPod.Store("192.0.2.1:8080", "test-pod") + resolver.podToWorkloadAndNamespace.Store("test-pod", "test-deployment@test-namespace") + err := resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + assert.Equal(t, "test-deployment", getStrAttr(attributes, "aws.remote.service", t)) + assert.Equal(t, "test-namespace", getStrAttr(attributes, "K8s.RemoteNamespace", t)) + + // Test case 2: "aws.remote.service" contains only IP + attributes = pcommon.NewMap() + attributes.PutStr("aws.remote.service", "192.0.2.2") + resourceAttributes = pcommon.NewMap() + resolver.ipToPod.Store("192.0.2.2", "test-pod-2") + resolver.podToWorkloadAndNamespace.Store("test-pod-2", "test-deployment-2@test-namespace-2") + err = resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + assert.Equal(t, "test-deployment-2", getStrAttr(attributes, "aws.remote.service", t)) + assert.Equal(t, "test-namespace-2", getStrAttr(attributes, "K8s.RemoteNamespace", t)) + + // Test case 3: "aws.remote.service" contains non-ip string + attributes = pcommon.NewMap() + attributes.PutStr("aws.remote.service", "not-an-ip") + resourceAttributes = pcommon.NewMap() + err = resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + assert.Equal(t, "not-an-ip", getStrAttr(attributes, "aws.remote.service", t)) + + // Test case 4 and 5: resourceAttributes contains "k8s.namespace.name" and EKS cluster name + attributes = pcommon.NewMap() + resourceAttributes = pcommon.NewMap() + resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") + resourceAttributes.PutStr("ec2.tag.kubernetes.io/cluster/test-cluster", "owned") + err = resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + assert.Equal(t, "test-namespace-3", getStrAttr(attributes, "K8s.Namespace", t)) + assert.Equal(t, "test-cluster", getStrAttr(attributes, "EKS.Cluster", t)) + + // Test case 6: Process with valid IP but GetWorkloadAndNamespaceByIP returns error + attributes = pcommon.NewMap() + attributes.PutStr("aws.remote.service", "192.168.1.2") + resourceAttributes = pcommon.NewMap() + err = resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + assert.Equal(t, "UnknownRemoteService", getStrAttr(attributes, "aws.remote.service", t)) + }) +} + +func TestExtractIPPort(t *testing.T) { + // Test valid IP:Port + ip, port, ok := extractIPPort("192.0.2.0:8080") + assert.Equal(t, "192.0.2.0", ip) + assert.Equal(t, "8080", port) + assert.True(t, ok) + + // Test invalid IP:Port + ip, port, ok = extractIPPort("192.0.2:8080") + assert.Equal(t, "", ip) + assert.Equal(t, "", port) + assert.False(t, ok) + + // Test IP only + ip, port, ok = extractIPPort("192.0.2.0") + assert.Equal(t, "", ip) + assert.Equal(t, "", port) + assert.False(t, ok) +} + +func TestGetHostNetworkPorts(t *testing.T) { + // Test Pod with no ports + pod := &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {}, + }, + }, + } + assert.Empty(t, getHostNetworkPorts(pod)) + + // Test Pod with one port + pod = &corev1.Pod{ + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + {HostPort: 8080}, + }, + }, + }, + }, + } + assert.Equal(t, []string{"8080"}, getHostNetworkPorts(pod)) + + // Test Pod with multiple ports + pod = &corev1.Pod{ + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + {HostPort: 8080}, + {HostPort: 8081}, + }, + }, + }, + }, + } + assert.Equal(t, []string{"8080", "8081"}, getHostNetworkPorts(pod)) +} + +func TestHandlePodUpdate(t *testing.T) { + testCases := []struct { + name string + oldPod *corev1.Pod + newPod *corev1.Pod + initialIPToPod map[string]string + expectedIPToPod map[string]string + }{ + { + name: "Old and New Pod Use Host Network, Different Ports", + oldPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + HostIP: "192.168.1.1", + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + { + HostPort: 8000, + }, + }, + }, + }, + }, + }, + newPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + HostIP: "192.168.1.1", + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + { + HostPort: 8080, + }, + }, + }, + }, + }, + }, + initialIPToPod: map[string]string{ + "192.168.1.1:8000": "mypod", + }, + expectedIPToPod: map[string]string{ + "192.168.1.1:8080": "mypod", + }, + }, + // ...other test cases... + { + name: "Old Pod Uses Host Network, New Pod Does Not", + oldPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + HostIP: "192.168.1.2", + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + { + HostPort: 8001, + }, + }, + }, + }, + }, + }, + newPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.1", + }, + Spec: corev1.PodSpec{ + HostNetwork: false, + }, + }, + initialIPToPod: map[string]string{ + "192.168.1.2:8001": "mypod", + }, + expectedIPToPod: map[string]string{ + "10.0.0.1": "mypod", + }, + }, + { + name: "Old Pod Does Not Use Host Network, New Pod Does", + oldPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.2", + }, + Spec: corev1.PodSpec{ + HostNetwork: false, + }, + }, + newPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + HostIP: "192.168.1.3", + }, + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{ + { + Ports: []corev1.ContainerPort{ + { + HostPort: 8002, + }, + }, + }, + }, + }, + }, + initialIPToPod: map[string]string{ + "10.0.0.2": "mypod", + }, + expectedIPToPod: map[string]string{ + "192.168.1.3:8002": "mypod", + }, + }, + { + name: "Old and New Pod Do Not Use Host Network, Different Pod IPs", + oldPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.3", + }, + Spec: corev1.PodSpec{ + HostNetwork: false, + }, + }, + newPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.4", + }, + Spec: corev1.PodSpec{ + HostNetwork: false, + }, + }, + initialIPToPod: map[string]string{ + "10.0.0.3": "mypod", + }, + expectedIPToPod: map[string]string{ + "10.0.0.4": "mypod", + }, + }, + { + name: "Old Pod Has Empty PodIP, New Pod Does Not Use Host Network, Non-Empty Pod IP", + oldPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + PodIP: "", + }, + Spec: corev1.PodSpec{ + HostNetwork: false, + }, + }, + newPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mypod", + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.5", + }, + Spec: corev1.PodSpec{ + HostNetwork: false, + }, + }, + initialIPToPod: map[string]string{}, + expectedIPToPod: map[string]string{ + "10.0.0.5": "mypod", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ipToPod := &sync.Map{} + // Initialize ipToPod map + for k, v := range tc.initialIPToPod { + ipToPod.Store(k, v) + } + handlePodUpdate(tc.newPod, tc.oldPod, ipToPod, mockDeleter) + + // Now validate that ipToPod map has been updated correctly + for key, expectedValue := range tc.expectedIPToPod { + val, ok := ipToPod.Load(key) + if !ok || val.(string) != expectedValue { + t.Errorf("Expected record for %v to be %v, got %v", key, expectedValue, val) + } + } + // Validate that old keys have been removed + for key := range tc.initialIPToPod { + if _, ok := tc.expectedIPToPod[key]; !ok { + if _, ok := ipToPod.Load(key); ok { + t.Errorf("Expected record for %v to be removed, but it was not", key) + } + } + } + }) + } +} diff --git a/processor/awsapmprocessor/processor.go b/processor/awsapmprocessor/processor.go new file mode 100644 index 0000000000..471292cf4b --- /dev/null +++ b/processor/awsapmprocessor/processor.go @@ -0,0 +1,268 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsapmprocessor + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" + + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/normalizer" + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/resolver" +) + +const ( + failedToProcessAttribute = "failed to Process attributes" + failedToProcessAttributeWithCustomRule = "failed to process attributes with custom rule, will drop it" +) + +// this is used to Process some attributes (like IP addresses) to a generic form to reduce high cardinality +type attributesMutator interface { + Process(attributes, resourceAttributes pcommon.Map, isTrace bool) error +} + +type customAllowlistMutator interface { + ShouldBeDropped(attributes, resourceAttributes pcommon.Map) (bool, error) +} + +type stopper interface { + Stop(context.Context) error +} + +type awsapmprocessor struct { + logger *zap.Logger + config *Config + customReplacer *customconfiguration.ReplaceActions + customAllowlistMutators []customAllowlistMutator + metricMutators []attributesMutator + traceMutators []attributesMutator + stoppers []stopper +} + +func (ap *awsapmprocessor) Start(_ context.Context, _ component.Host) error { + attributesResolver := resolver.NewAttributesResolver(ap.config.Resolvers, ap.logger) + ap.stoppers = append(ap.stoppers, attributesResolver) + ap.metricMutators = append(ap.metricMutators, attributesResolver) + + attributesNormalizer := normalizer.NewAttributesNormalizer(ap.logger) + ap.metricMutators = append(ap.metricMutators, attributesNormalizer) + + ap.customReplacer = customconfiguration.NewCustomReplacer(ap.config.Rules) + ap.traceMutators = append(ap.traceMutators, attributesResolver, attributesNormalizer, ap.customReplacer) + + customKeeper := customconfiguration.NewCustomKeeper(ap.config.Rules) + ap.customAllowlistMutators = append(ap.customAllowlistMutators, customKeeper) + customDropper := customconfiguration.NewCustomDropper(ap.config.Rules) + ap.customAllowlistMutators = append(ap.customAllowlistMutators, customDropper) + + return nil +} + +func (ap *awsapmprocessor) Shutdown(ctx context.Context) error { + for _, stopper := range ap.stoppers { + err := stopper.Stop(ctx) + if err != nil { + ap.logger.Error("failed to stop", zap.Error(err)) + } + } + return nil +} + +func (ap *awsapmprocessor) processTraces(ctx context.Context, td ptrace.Traces) (ptrace.Traces, error) { + rss := td.ResourceSpans() + for i := 0; i < rss.Len(); i++ { + rs := rss.At(i) + ilss := rs.ScopeSpans() + resourceAttributes := rs.Resource().Attributes() + for j := 0; j < ilss.Len(); j++ { + ils := ilss.At(j) + spans := ils.Spans() + for k := 0; k < spans.Len(); k++ { + span := spans.At(k) + for _, Mutator := range ap.traceMutators { + err := Mutator.Process(span.Attributes(), resourceAttributes, true) + if err != nil { + ap.logger.Debug("failed to Process span", zap.Error(err)) + } + } + } + } + } + return td, nil +} + +func (ap *awsapmprocessor) processMetrics(ctx context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { + rms := md.ResourceMetrics() + for i := 0; i < rms.Len(); i++ { + rs := rms.At(i) + ilms := rs.ScopeMetrics() + resourceAttributes := rs.Resource().Attributes() + for j := 0; j < ilms.Len(); j++ { + ils := ilms.At(j) + metrics := ils.Metrics() + for k := 0; k < metrics.Len(); k++ { + m := metrics.At(k) + ap.processMetricAttributes(ctx, m, resourceAttributes) + } + } + } + return md, nil +} + +// Attributes are provided for each log and trace, but not at the metric level +// Need to process attributes for every data point within a metric. +func (ap *awsapmprocessor) processMetricAttributes(ctx context.Context, m pmetric.Metric, resourceAttribes pcommon.Map) { + + // This is a lot of repeated code, but since there is no single parent superclass + // between metric data types, we can't use polymorphism. + switch m.Type() { + case pmetric.MetricTypeGauge: + dps := m.Gauge().DataPoints() + for i := 0; i < dps.Len(); i++ { + for _, Mutator := range ap.metricMutators { + err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + } + dps.RemoveIf(func(d pmetric.NumberDataPoint) bool { + for _, Mutator := range ap.customAllowlistMutators { + shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + if err != nil { + ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + return true + } else if shouldBeDropped { + return true + } + } + return false + }) + for i := 0; i < dps.Len(); i++ { + err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + case pmetric.MetricTypeSum: + dps := m.Sum().DataPoints() + for i := 0; i < dps.Len(); i++ { + for _, Mutator := range ap.metricMutators { + err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + } + dps.RemoveIf(func(d pmetric.NumberDataPoint) bool { + for _, Mutator := range ap.customAllowlistMutators { + shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + if err != nil { + ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + return true + } else if shouldBeDropped { + return true + } + } + return false + }) + for i := 0; i < dps.Len(); i++ { + err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + case pmetric.MetricTypeHistogram: + dps := m.Histogram().DataPoints() + for i := 0; i < dps.Len(); i++ { + for _, Mutator := range ap.metricMutators { + err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + } + dps.RemoveIf(func(d pmetric.HistogramDataPoint) bool { + for _, Mutator := range ap.customAllowlistMutators { + shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + if err != nil { + ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + return true + } else if shouldBeDropped { + return true + } + } + return false + }) + for i := 0; i < dps.Len(); i++ { + err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + case pmetric.MetricTypeExponentialHistogram: + dps := m.ExponentialHistogram().DataPoints() + for i := 0; i < dps.Len(); i++ { + for _, Mutator := range ap.metricMutators { + err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + } + dps.RemoveIf(func(d pmetric.ExponentialHistogramDataPoint) bool { + for _, Mutator := range ap.customAllowlistMutators { + shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + if err != nil { + ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + return true + } else if shouldBeDropped { + return true + } + } + return false + }) + for i := 0; i < dps.Len(); i++ { + err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + case pmetric.MetricTypeSummary: + dps := m.Summary().DataPoints() + for i := 0; i < dps.Len(); i++ { + for _, Mutator := range ap.metricMutators { + err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + } + dps.RemoveIf(func(d pmetric.SummaryDataPoint) bool { + for _, Mutator := range ap.customAllowlistMutators { + shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + if err != nil { + ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) + return true + } else if shouldBeDropped { + return true + } + } + return false + }) + for i := 0; i < dps.Len(); i++ { + err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + if err != nil { + ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) + } + } + default: + ap.logger.Info("Ignore unknown metric type", zap.String("type", m.Type().String())) + } +} diff --git a/processor/awsapmprocessor/testdata/config.yaml b/processor/awsapmprocessor/testdata/config.yaml new file mode 100644 index 0000000000..d3b1ae6509 --- /dev/null +++ b/processor/awsapmprocessor/testdata/config.yaml @@ -0,0 +1,29 @@ +awsapm: + resolvers: [eks] + rules: + - selectors: + - dimension: Operation + match: "* /api/visits/*" + - dimension: RemoteOperation + match: "*" + action: keep + rule_name: "keep01" + + - selectors: + - dimension: RemoteService + match: "UnknownRemoteService" + - dimension: RemoteOperation + match: "GetShardIterator" + action: drop + + - selectors: + - dimension: Operation + match: "* /api/visits/*" + - dimension: RemoteOperation + match: "*" + replacements: + - target_dimension: RemoteOperation + value: ListPetsByCustomer + - target_dimension: ResourceTarget + value: ' ' + action: replace \ No newline at end of file From 42b7f3f7914017e14bbe968d0b992f7097345bd4 Mon Sep 17 00:00:00 2001 From: Jason Polanco Date: Thu, 28 Sep 2023 19:43:31 -0400 Subject: [PATCH 02/38] go mod update for apm branch after merge --- go.sum | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/go.sum b/go.sum index 3f509b0aff..5133bc9bfd 100644 --- a/go.sum +++ b/go.sum @@ -25,28 +25,27 @@ cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aD cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= -cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= -cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/bigquery v1.53.0 h1:K3wLbjbnSlxhuG5q4pntHv5AEbQM1QqHKGYgwFIqOTg= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 h1:aRVqY1p2IJaBGStWMsQMpkAa83cPkCDLl80eOj0Rbz4= +cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68/go.mod h1:1a3eRNYX12fs5UABBIXS8HXVvQbX9hRB/RkEBPORpe8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= -cloud.google.com/go/monitoring v1.13.0 h1:2qsrgXGVoRXpP7otZ14eE1I568zAa92sJSDPyOJvwjM= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.30.0 h1:vCge8m7aUKBJYOgrZp7EsNDf6QMd2CAlXZqWTn3yq6s= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= From 96d80aeff0680e1ff51bcdab11f042d879134931 Mon Sep 17 00:00:00 2001 From: Jason Polanco Date: Fri, 6 Oct 2023 16:44:43 -0400 Subject: [PATCH 03/38] Merge APM translation changes into main-apm branch (#517) Co-authored-by: Lisa Guo Co-authored-by: Lisa Guo Co-authored-by: Harry Co-authored-by: Hyunsoo Kim <884273+movence@users.noreply.github.com> --- .github/workflows/PR-build.yml | 8 +- .github/workflows/apm-beta-pre-release.yml | 19 + go.mod | 14 +- go.sum | 20 +- processor/awsapmprocessor/config.go | 2 +- processor/awsapmprocessor/config_test.go | 2 +- .../customconfiguration/common.go | 0 .../customconfiguration/dropper.go | 0 .../customconfiguration/dropper_test.go | 0 .../customconfiguration/keeper.go | 0 .../customconfiguration/keeper_test.go | 0 .../customconfiguration/replacer.go | 0 .../customconfiguration/replacer_test.go | 0 processor/awsapmprocessor/processor.go | 2 +- service/defaultcomponents/components.go | 9 +- service/defaultcomponents/components_test.go | 7 +- translator/config/schema.json | 83 ++- .../apm_and_kubernetes_config.conf | 27 + .../apm_and_kubernetes_config.json | 24 + .../apm_and_kubernetes_config.yaml | 646 ++++++++++++++++++ .../sampleConfig/base_apm_config.conf | 27 + .../sampleConfig/base_apm_config.json | 17 + .../sampleConfig/base_apm_config.yaml | 490 +++++++++++++ translator/tocwconfig/tocwconfig_test.go | 22 +- translator/translate/otel/common/common.go | 27 + .../exporter/awsemf/awsemf_default_apm.yaml | 34 + .../otel/exporter/awsemf/translator.go | 9 + .../otel/exporter/awsemf/translator_test.go | 59 ++ .../exporter/awsxray/awsxray_default_apm.yaml | 2 + .../otel/exporter/awsxray/translator.go | 16 + .../otel/exporter/awsxray/translator_test.go | 27 + .../otel/extension/awsproxy/translator.go | 37 + .../extension/awsproxy/translator_test.go | 26 + .../translate/otel/pipeline/apm/translator.go | 64 ++ .../otel/pipeline/apm/translator_test.go | 117 ++++ .../processor/awsapm/testdata/config.yaml | 1 + .../awsapm/testdata/invalidRulesConfig.json | 20 + .../awsapm/testdata/validRulesConfig.json | 63 ++ .../awsapm/testdata/validRulesConfig.yaml | 28 + .../otel/processor/awsapm/translator.go | 119 ++++ .../otel/processor/awsapm/translator_test.go | 80 +++ .../resourcedetection/configs/config.yaml | 6 + .../processor/resourcedetection/translator.go | 61 ++ .../resourcedetection/translator_test.go | 60 ++ .../otel/receiver/otlp/apm_config.yaml | 5 + .../otel/receiver/otlp/translator.go | 20 +- .../otel/receiver/otlp/translator_test.go | 44 ++ translator/translate/otel/translate_otel.go | 3 + .../translate/otel/translate_otel_test.go | 32 + 49 files changed, 2359 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/apm-beta-pre-release.yml rename processor/awsapmprocessor/{internal => }/customconfiguration/common.go (100%) rename processor/awsapmprocessor/{internal => }/customconfiguration/dropper.go (100%) rename processor/awsapmprocessor/{internal => }/customconfiguration/dropper_test.go (100%) rename processor/awsapmprocessor/{internal => }/customconfiguration/keeper.go (100%) rename processor/awsapmprocessor/{internal => }/customconfiguration/keeper_test.go (100%) rename processor/awsapmprocessor/{internal => }/customconfiguration/replacer.go (100%) rename processor/awsapmprocessor/{internal => }/customconfiguration/replacer_test.go (100%) create mode 100644 translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.conf create mode 100644 translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.json create mode 100644 translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml create mode 100644 translator/tocwconfig/sampleConfig/base_apm_config.conf create mode 100644 translator/tocwconfig/sampleConfig/base_apm_config.json create mode 100644 translator/tocwconfig/sampleConfig/base_apm_config.yaml create mode 100644 translator/translate/otel/exporter/awsemf/awsemf_default_apm.yaml create mode 100644 translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml create mode 100644 translator/translate/otel/extension/awsproxy/translator.go create mode 100644 translator/translate/otel/extension/awsproxy/translator_test.go create mode 100644 translator/translate/otel/pipeline/apm/translator.go create mode 100644 translator/translate/otel/pipeline/apm/translator_test.go create mode 100644 translator/translate/otel/processor/awsapm/testdata/config.yaml create mode 100644 translator/translate/otel/processor/awsapm/testdata/invalidRulesConfig.json create mode 100644 translator/translate/otel/processor/awsapm/testdata/validRulesConfig.json create mode 100644 translator/translate/otel/processor/awsapm/testdata/validRulesConfig.yaml create mode 100644 translator/translate/otel/processor/awsapm/translator.go create mode 100644 translator/translate/otel/processor/awsapm/translator_test.go create mode 100644 translator/translate/otel/processor/resourcedetection/configs/config.yaml create mode 100644 translator/translate/otel/processor/resourcedetection/translator.go create mode 100644 translator/translate/otel/processor/resourcedetection/translator_test.go create mode 100644 translator/translate/otel/receiver/otlp/apm_config.yaml diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml index 9aca3ef1a1..9bada91b1d 100644 --- a/.github/workflows/PR-build.yml +++ b/.github/workflows/PR-build.yml @@ -5,9 +5,9 @@ name: PR Build on: workflow_dispatch: pull_request: - branches: + branches: - main* - types: + types: - opened - synchronize - reopened @@ -47,7 +47,7 @@ jobs: if: needs.changes.outputs.lint == 'true' uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 cache: false - name: Check out code @@ -102,7 +102,7 @@ jobs: if: needs.changes.outputs.build == 'true' uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 cache: false - name: Check out code diff --git a/.github/workflows/apm-beta-pre-release.yml b/.github/workflows/apm-beta-pre-release.yml new file mode 100644 index 0000000000..1888325a11 --- /dev/null +++ b/.github/workflows/apm-beta-pre-release.yml @@ -0,0 +1,19 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +name: APM Beta Pre-Release +on: + push: + branches: + - main-apm +jobs: + BuildAndUpload: + uses: ./.github/workflows/test-build.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + ContainerRepositoryNameAndTag: "apm-beta-pre-release:latest" + BucketKey: "apm-beta-pre-release" + PackageBucketKey: "apm-beta-pre-release" diff --git a/go.mod b/go.mod index 4b52f38a61..8a8785973b 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,10 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxr replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713 + +replace go.opentelemetry.io/collector/config/confighttp => github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713 + // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 replace github.com/shirou/gopsutil/v3 => github.com/aws/telegraf/patches/gopsutil/v3 v3.0.0-20230915153624-7629361f8380 // indirect @@ -94,7 +98,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/eks v1.27.15 github.com/aws/smithy-go v1.13.5 github.com/bigkevmcd/go-configparser v0.0.0-20200217161103-d137835d2579 - github.com/deckarep/golang-set/v2 v2.3.1 github.com/go-kit/log v0.2.1 github.com/gobwas/glob v0.2.3 github.com/google/cadvisor v0.47.3 // indirect @@ -148,7 +151,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 gopkg.in/yaml.v3 v3.0.1 - gotest.tools/v3 v3.1.0 + gotest.tools/v3 v3.2.0 k8s.io/api v0.28.1 k8s.io/apimachinery v0.28.1 k8s.io/client-go v0.28.1 @@ -156,6 +159,9 @@ require ( ) require ( + github.com/deckarep/golang-set/v2 v2.3.1 + github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy v0.84.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.84.0 go.opentelemetry.io/collector/config/configtelemetry v0.84.1-0.20230908201109-ab3d6c5b6470 go.opentelemetry.io/collector/extension v0.84.1-0.20230908201109-ab3d6c5b6470 go.opentelemetry.io/collector/processor v0.84.1-0.20230908201109-ab3d6c5b6470 @@ -175,7 +181,9 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.19.1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Showmax/go-fqdn v1.0.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/alecthomas/participle v0.4.1 // indirect github.com/alecthomas/participle/v2 v2.0.0 // indirect @@ -306,6 +314,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs v0.84.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy v0.84.0 // indirect @@ -315,6 +324,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubelet v0.84.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.84.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.84.0 // indirect diff --git a/go.sum b/go.sum index 5133bc9bfd..4c64eb2f2c 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.19.1 h1:LyRJCTBJP53P1JURFbhFSRz36gxaBtMAjzjlYupNR7Q= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.19.1/go.mod h1:Xx0VKh7GJ4si3rmElbh19Mejxz68ibWg/J30ZOMrqzU= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= @@ -115,6 +117,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.32.0 h1:P+RUjEaRU0GMMbYexGMDyrMkLhbbBVUVISDywi+IlFU= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM= +github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -139,6 +143,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= +github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713 h1:ACWoE8NqyI13oKdpg+074JIta5wmxsL1kOirTzDO5Gk= +github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713/go.mod h1:4g9P7MZPReFnNJD0lpavI/LR0vIwlsTJov+hJoKT+nM= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20230928170322-0df38c533713 h1:wt44oIHsyO9tNSw36V2oWZG4cTzrpqBe7JKyfXHBa5w= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20230928170322-0df38c533713/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20230928170322-0df38c533713 h1:/DboazGzxalMCGykP//7s3m1+YwUaXydlUzX8uIrXUg= @@ -161,6 +167,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0 github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20230928170322-0df38c533713 h1:SZf6K08K4cyoGDdkPthRGcsT1Gk6Z7o6QN01oM8Jr6A= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20230928170322-0df38c533713/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713 h1:tKGat0aoXPkscaShYYRbnXH14asXqi1Iem4K3nMrNpk= +github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713/go.mod h1:fbCDqcaNUzfvbpI4y91hT8UfV18VyIKfS42BsPRDAuc= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20230928170322-0df38c533713 h1:hDrhgnYst1yL2CLUmlwK1MqH4iqjLfxWZpw85X8cojE= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20230928170322-0df38c533713/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20230928170322-0df38c533713 h1:YPlfoYRq2+HxqibbpC3W+Go4WWJzVKEI5rziSHg/5Fc= @@ -955,7 +963,11 @@ github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je4 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.84.0 h1:WFCStS52a3bMG9nEG6Eo8+EfgtXfGclTN/H4as7jdkE= +github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy v0.84.0 h1:ysq9+0eESy8Dj7Yp3Ijn6uDAtvR+h+LoTwzZbw78gRc= +github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy v0.84.0/go.mod h1:CwZNGQC/yVWCJrltkkGGGPeUUqtWuLQyADWfmqfol+Q= github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage v0.84.0 h1:elXwC0d93Y4/t5nZmSVVX6EH9GmGgKubNCFbu3sjxEc= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.84.0 h1:BWVaklpLBrFOLlW/nqT3o2onEhaMIOr2/LBnEciDVjQ= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.84.0/go.mod h1:e/k1LzterZSHvxWF3HG9Wo2VtuVrYAt0DE+inTVaD+g= github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics v0.84.0 h1:KwJDgnqegui18ebzCfxHURRstPjY2CfM/yGRugcdOT8= github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/metrics v0.84.0/go.mod h1:WAScm+oitM87OWSy+pPAC6eCzg3xhYz3VBSef2+zV60= github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy v0.84.0 h1:vfZIgsfOkrY+bqh3HGYWDtvxN5SR4B+IYJqY1733j+8= @@ -970,6 +982,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0. github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig v0.84.0/go.mod h1:iL+tGP94Xdes4iUmss/Me8OOqvJhWeBWPcW8OgHQZyo= github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubelet v0.84.0 h1:ezuTl9JDE/v83DnyJk5jLEumkgu0nxdoWXNTKmJ/+KA= github.com/open-telemetry/opentelemetry-collector-contrib/internal/kubelet v0.84.0/go.mod h1:u8PmrJN1vhdnUtyMkxoze478uOzX/bVTeJebvM5xyHU= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders v0.84.0 h1:4xV4X+Zze+nNLkyK35LkD1AOL/V7qxgptsJLEWfxhj4= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders v0.84.0/go.mod h1:rcxnHaBL7R/VoO8yahgIFL7NZIdFaakl4Jpy2tbNeGc= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.84.0 h1:hCHJbiLdDwsswhfllFCq4fjRWUCz2GAHhzOB1n7jHK4= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.84.0/go.mod h1:0us3rAudWnHES0nOtmTsUjgQtlKJyiozaC5osgJM7cU= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.84.0 h1:NXzx/YViPbapdfI0RZ2RJk4XgKU099Ci9rGBsZ/div8= @@ -1256,8 +1270,6 @@ go.opentelemetry.io/collector/config/configcompression v0.84.1-0.20230908201109- go.opentelemetry.io/collector/config/configcompression v0.84.1-0.20230908201109-ab3d6c5b6470/go.mod h1:LaavoxZsro5lL7qh1g9DMifG0qixWPEecW18Qr8bpag= go.opentelemetry.io/collector/config/configgrpc v0.84.0 h1:wWYWbmm8EHm33Xllfe1o+AEmVHbjx4A5bDSx9TfM3bc= go.opentelemetry.io/collector/config/configgrpc v0.84.0/go.mod h1:cs/g9nIJZh/R6G8XoBIO8chtU0RoiaG3DcP454pKenY= -go.opentelemetry.io/collector/config/confighttp v0.84.1-0.20230908201109-ab3d6c5b6470 h1:r93l0jZH/52k6Etzk3s0emCbzZNHZHRkD4AZQVVBXnI= -go.opentelemetry.io/collector/config/confighttp v0.84.1-0.20230908201109-ab3d6c5b6470/go.mod h1:YUrduvwwO7zNGS9V2VrVof2R6liY/TekAyF+tkaRJm4= go.opentelemetry.io/collector/config/confignet v0.84.1-0.20230908201109-ab3d6c5b6470 h1:QOIvMZc5E8KT+1r65KxohRtjjn3hac+YF5nnnGbTJc4= go.opentelemetry.io/collector/config/confignet v0.84.1-0.20230908201109-ab3d6c5b6470/go.mod h1:cpO8JYWGONaViOygKVw+Hd2UoBcn2cUiyi0WWeFTwJY= go.opentelemetry.io/collector/config/configopaque v0.84.1-0.20230908201109-ab3d6c5b6470 h1:t6LXR0S4c3FfXGJ/hbwqdHeX7I4G6TOc83SPqGCkKPE= @@ -1920,8 +1932,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= -gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= +gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= +gotest.tools/v3 v3.2.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/processor/awsapmprocessor/config.go b/processor/awsapmprocessor/config.go index 74772b1195..19572e01cf 100644 --- a/processor/awsapmprocessor/config.go +++ b/processor/awsapmprocessor/config.go @@ -4,7 +4,7 @@ package awsapmprocessor import ( - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/customconfiguration" ) type Config struct { diff --git a/processor/awsapmprocessor/config_test.go b/processor/awsapmprocessor/config_test.go index ec6bfd223e..36c0e142c5 100644 --- a/processor/awsapmprocessor/config_test.go +++ b/processor/awsapmprocessor/config_test.go @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap/confmaptest" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/customconfiguration" ) func TestLoadConfig(t *testing.T) { diff --git a/processor/awsapmprocessor/internal/customconfiguration/common.go b/processor/awsapmprocessor/customconfiguration/common.go similarity index 100% rename from processor/awsapmprocessor/internal/customconfiguration/common.go rename to processor/awsapmprocessor/customconfiguration/common.go diff --git a/processor/awsapmprocessor/internal/customconfiguration/dropper.go b/processor/awsapmprocessor/customconfiguration/dropper.go similarity index 100% rename from processor/awsapmprocessor/internal/customconfiguration/dropper.go rename to processor/awsapmprocessor/customconfiguration/dropper.go diff --git a/processor/awsapmprocessor/internal/customconfiguration/dropper_test.go b/processor/awsapmprocessor/customconfiguration/dropper_test.go similarity index 100% rename from processor/awsapmprocessor/internal/customconfiguration/dropper_test.go rename to processor/awsapmprocessor/customconfiguration/dropper_test.go diff --git a/processor/awsapmprocessor/internal/customconfiguration/keeper.go b/processor/awsapmprocessor/customconfiguration/keeper.go similarity index 100% rename from processor/awsapmprocessor/internal/customconfiguration/keeper.go rename to processor/awsapmprocessor/customconfiguration/keeper.go diff --git a/processor/awsapmprocessor/internal/customconfiguration/keeper_test.go b/processor/awsapmprocessor/customconfiguration/keeper_test.go similarity index 100% rename from processor/awsapmprocessor/internal/customconfiguration/keeper_test.go rename to processor/awsapmprocessor/customconfiguration/keeper_test.go diff --git a/processor/awsapmprocessor/internal/customconfiguration/replacer.go b/processor/awsapmprocessor/customconfiguration/replacer.go similarity index 100% rename from processor/awsapmprocessor/internal/customconfiguration/replacer.go rename to processor/awsapmprocessor/customconfiguration/replacer.go diff --git a/processor/awsapmprocessor/internal/customconfiguration/replacer_test.go b/processor/awsapmprocessor/customconfiguration/replacer_test.go similarity index 100% rename from processor/awsapmprocessor/internal/customconfiguration/replacer_test.go rename to processor/awsapmprocessor/customconfiguration/replacer_test.go diff --git a/processor/awsapmprocessor/processor.go b/processor/awsapmprocessor/processor.go index 471292cf4b..18b37fe682 100644 --- a/processor/awsapmprocessor/processor.go +++ b/processor/awsapmprocessor/processor.go @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/customconfiguration" "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/normalizer" "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/resolver" ) diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index 0df6e4818d..3afe64f731 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -7,8 +7,10 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter" + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver" @@ -25,6 +27,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/plugins/outputs/cloudwatch" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger" + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor" ) func Factories() (otelcol.Factories, error) { @@ -42,10 +45,12 @@ func Factories() (otelcol.Factories, error) { } if factories.Processors, err = processor.MakeFactoryMap( + awsapmprocessor.NewFactory(), batchprocessor.NewFactory(), cumulativetodeltaprocessor.NewFactory(), ec2tagger.NewFactory(), metricstransformprocessor.NewFactory(), + resourcedetectionprocessor.NewFactory(), transformprocessor.NewFactory(), ); err != nil { return otelcol.Factories{}, err @@ -61,7 +66,9 @@ func Factories() (otelcol.Factories, error) { return otelcol.Factories{}, err } - if factories.Extensions, err = extension.MakeFactoryMap(); err != nil { + if factories.Extensions, err = extension.MakeFactoryMap( + awsproxy.NewFactory(), + ); err != nil { return otelcol.Factories{}, err } diff --git a/service/defaultcomponents/components_test.go b/service/defaultcomponents/components_test.go index 01f93fbd9f..a555a77852 100644 --- a/service/defaultcomponents/components_test.go +++ b/service/defaultcomponents/components_test.go @@ -11,9 +11,9 @@ import ( const ( receiversCount = 5 - processorCount = 5 + processorCount = 7 exportersCount = 5 - extensionsCount = 0 + extensionsCount = 1 ) func TestComponents(t *testing.T) { @@ -29,11 +29,13 @@ func TestComponents(t *testing.T) { processors := factories.Processors assert.Len(t, processors, processorCount) + assert.NotNil(t, processors["awsapm"]) assert.NotNil(t, processors["batch"]) assert.NotNil(t, processors["cumulativetodelta"]) assert.NotNil(t, processors["ec2tagger"]) assert.NotNil(t, processors["metricstransform"]) assert.NotNil(t, processors["transform"]) + assert.NotNil(t, processors["resourcedetection"]) exporters := factories.Exporters assert.Len(t, exporters, exportersCount) @@ -45,4 +47,5 @@ func TestComponents(t *testing.T) { extensions := factories.Extensions assert.Len(t, extensions, extensionsCount) + assert.NotNil(t, extensions["awsproxy"]) } diff --git a/translator/config/schema.json b/translator/config/schema.json index 656ec61d1a..97ab8003ec 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -511,6 +511,82 @@ "metrics_collected": { "type": "object", "properties": { + "apm": { + "type": "object", + "properties": { + "rules": { + "description": "Custom rules defined by customer", + "type": "array", + "items": { + "type": "object", + "properties": { + "selectors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dimension": { + "description": "dimension used for matching", + "type": "string", + "minLength": 1 + }, + "match": { + "description": "regex used for match", + "type": "string", + "minLength": 1 + } + }, + "required": [ + "dimension", + "match" + ] + } + }, + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "target_dimension": { + "description": "dimension to be replaced", + "type": "string", + "minLength": 1 + }, + "value": { + "description": "replacement value", + "type": "string" + } + }, + "required": [ + "target_dimension", + "value" + ] + } + }, + "action": { + "description": "action to be done, either keep, drop or replace", + "type": "string", + "enum": [ + "drop", + "keep", + "replace" + ] + }, + "rule_name": { + "description": "name of rule", + "type": "string", + "minLength": 1 + } + }, + "required": [ + "selectors", + "action" + ] + } + } + }, + "additionalProperties": true + }, "ecs": { "type": "object", "properties": { @@ -759,7 +835,7 @@ "retentionInDaysDefinition": { "type": "integer", "enum": [ - -1, + -1, 1, 3, 5, @@ -812,6 +888,11 @@ "traces_collected": { "type": "object", "properties": { + "apm": { + "type": "object", + "properties": {}, + "additionalProperties": true + }, "xray": { "$ref": "#/definitions/tracesDefinition/definitions/xrayDefinition" }, diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.conf b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.conf new file mode 100644 index 0000000000..007bb60efb --- /dev/null +++ b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.conf @@ -0,0 +1,27 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "host_name_from_env" + interval = "60s" + logfile = "" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + +[outputs] + + [[outputs.cloudwatchlogs]] + endpoint_override = "https://fake_endpoint" + force_flush_interval = "5s" + log_stream_name = "host_name_from_env" + region = "us-east-1" + +[processors] diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.json b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.json new file mode 100644 index 0000000000..d30415469f --- /dev/null +++ b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.json @@ -0,0 +1,24 @@ +{ + "agent": { + "region": "us-east-1" + }, + "logs": { + "metrics_collected": { + "apm": { + }, + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30, + "disable_metric_extraction": true, + "enhanced_container_insights": false + } + }, + "force_flush_interval": 5, + "endpoint_override":"https://fake_endpoint" + }, + "traces": { + "traces_collected": { + "apm": {} + } + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml new file mode 100644 index 0000000000..1f01de5964 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml @@ -0,0 +1,646 @@ +connectors: {} +exporters: + awsemf/apm: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: true + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - Namespace + - Service + - - ClusterName + - Namespace + label_matchers: [] + metric_name_selectors: + - pod_cpu_utilization + - pod_memory_utilization + - pod_network_rx_bytes + - pod_network_tx_bytes + - pod_cpu_utilization_over_pod_limit + - pod_memory_utilization_over_pod_limit + - dimensions: + - - ClusterName + - Namespace + - PodName + label_matchers: [] + metric_name_selectors: + - pod_number_of_container_restarts + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - pod_cpu_reserved_capacity + - pod_memory_reserved_capacity + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_cpu_utilization + - node_memory_utilization + - node_network_total_bytes + - node_cpu_reserved_capacity + - node_memory_reserved_capacity + - node_number_of_running_pods + - node_number_of_running_containers + - dimensions: + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_cpu_usage_total + - node_cpu_limit + - node_memory_working_set + - node_memory_limit + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_filesystem_utilization + - dimensions: + - - ClusterName + - Namespace + - Service + - - ClusterName + label_matchers: [] + metric_name_selectors: + - service_number_of_running_pods + - dimensions: + - - ClusterName + - Namespace + - - ClusterName + label_matchers: [] + metric_name_selectors: + - namespace_number_of_running_pods + - dimensions: + - - ClusterName + label_matchers: [] + metric_name_selectors: + - cluster_node_count + - cluster_failed_node_count + metric_descriptors: [] + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + shared_credentials_file: [] + version: "0" + awsemf/containerinsights: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: true + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - Namespace + - Service + - - ClusterName + - Namespace + label_matchers: [] + metric_name_selectors: + - pod_cpu_utilization + - pod_memory_utilization + - pod_network_rx_bytes + - pod_network_tx_bytes + - pod_cpu_utilization_over_pod_limit + - pod_memory_utilization_over_pod_limit + - dimensions: + - - ClusterName + - Namespace + - PodName + label_matchers: [] + metric_name_selectors: + - pod_number_of_container_restarts + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - pod_cpu_reserved_capacity + - pod_memory_reserved_capacity + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_cpu_utilization + - node_memory_utilization + - node_network_total_bytes + - node_cpu_reserved_capacity + - node_memory_reserved_capacity + - node_number_of_running_pods + - node_number_of_running_containers + - dimensions: + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_cpu_usage_total + - node_cpu_limit + - node_memory_working_set + - node_memory_limit + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_filesystem_utilization + - dimensions: + - - ClusterName + - Namespace + - Service + - - ClusterName + label_matchers: [] + metric_name_selectors: + - service_number_of_running_pods + - dimensions: + - - ClusterName + - Namespace + - - ClusterName + label_matchers: [] + metric_name_selectors: + - namespace_number_of_running_pods + - dimensions: + - - ClusterName + label_matchers: [] + metric_name_selectors: + - cluster_node_count + - cluster_failed_node_count + metric_descriptors: [] + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + shared_credentials_file: [] + version: "0" + awsxray/apm: + aws_log_groups: [] + certificate_file_path: "" + endpoint: "" + imds_retries: 1 + index_all_attributes: false + indexed_attributes: + - aws.local.service + - aws.local.operation + - aws.remote.service + - aws.remote.operation + - EKS.Cluster + - K8s.Namespace + - K8s.RemoteNamespace + - aws.remote.target + local_mode: false + max_retries: 2 + no_verify_ssl: false + num_workers: 8 + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + role_arn: "" + shared_credentials_file: [] + telemetry: + enabled: true + include_metadata: true +extensions: + awsproxy/apm: + aws_endpoint: "" + endpoint: 0.0.0.0:2000 + local_mode: false + proxy_address: "" + region: "" + role_arn: "" +processors: + awsapm: + resolvers: + - eks + rules: [] + batch/containerinsights: + metadata_cardinality_limit: 1000 + metadata_keys: [] + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + resourcedetection: + aks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + attributes: [] + auth: null + azure: + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + compression: "" + consul: + address: "" + datacenter: "" + meta: {} + namespace: "" + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + token: '[REDACTED]' + token_file: "" + detectors: + - eks + - env + - ec2 + docker: + resource_attributes: + host.name: + enabled: true + os.type: + enabled: true + ec2: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.image.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + tags: + - ^kubernetes.io/cluster/.*$ + ecs: + resource_attributes: + aws.ecs.cluster.arn: + enabled: true + aws.ecs.launchtype: + enabled: true + aws.ecs.task.arn: + enabled: true + aws.ecs.task.family: + enabled: true + aws.ecs.task.revision: + enabled: true + aws.log.group.arns: + enabled: true + aws.log.group.names: + enabled: true + aws.log.stream.arns: + enabled: true + aws.log.stream.names: + enabled: true + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + eks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + elasticbeanstalk: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + deployment.environment: + enabled: true + service.instance.id: + enabled: true + service.version: + enabled: true + endpoint: "" + gcp: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.id: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + gcp.cloud_run.job.execution: + enabled: true + gcp.cloud_run.job.task_index: + enabled: true + gcp.gce.instance.hostname: + enabled: false + gcp.gce.instance.name: + enabled: false + host.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + k8s.cluster.name: + enabled: true + headers: {} + heroku: + resource_attributes: + cloud.provider: + enabled: true + heroku.app.id: + enabled: true + heroku.dyno.id: + enabled: true + heroku.release.commit: + enabled: true + heroku.release.creation_timestamp: + enabled: true + service.instance.id: + enabled: true + service.name: + enabled: true + service.version: + enabled: true + idle_conn_timeout: 1m30s + lambda: + resource_attributes: + aws.log.group.names: + enabled: true + aws.log.stream.names: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.instance: + enabled: true + faas.max_memory: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + max_conns_per_host: null + max_idle_conns: 100 + max_idle_conns_per_host: null + openshift: + address: "" + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + k8s.cluster.name: + enabled: true + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + token: "" + override: false + read_buffer_size: 0 + system: + hostname_sources: [] + resource_attributes: + host.arch: + enabled: false + host.id: + enabled: false + host.name: + enabled: true + os.description: + enabled: false + os.type: + enabled: true + timeout: 2s + write_buffer_size: 0 +receivers: + awscontainerinsightreceiver: + add_container_name_metric_label: false + add_full_pod_name_metric_label: false + add_service_as_attribute: true + certificate_file_path: "" + cluster_name: TestCluster + collection_interval: 30s + container_orchestrator: eks + enable_control_plane_metrics: false + endpoint: "" + imds_retries: 1 + leader_lock_name: cwagent-clusterleader + leader_lock_using_config_map_only: true + local_mode: false + max_retries: 0 + no_verify_ssl: false + num_workers: 0 + prefer_full_pod_name: false + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 0 + resource_arn: "" + role_arn: "" + shared_credentials_file: [] + otlp/apm: + protocols: + grpc: + auth: null + endpoint: 0.0.0.0:4317 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp + write_buffer_size: 0 + http: + auth: null + cors: null + endpoint: 0.0.0.0:4318 + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + response_headers: {} + tls: null + traces_url_path: /v1/traces +service: + extensions: + - awsproxy/apm + pipelines: + metrics/apm: + exporters: + - awsemf/apm + processors: + - resourcedetection + - awsapm + receivers: + - otlp/apm + metrics/containerinsights: + exporters: + - awsemf/containerinsights + processors: + - batch/containerinsights + receivers: + - awscontainerinsightreceiver + traces/apm: + exporters: + - awsxray/apm + processors: + - resourcedetection + - awsapm + receivers: + - otlp/apm + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: [] + initial_fields: {} + level: info + output_paths: [] + sampling: + initial: 2 + thereafter: 500 + metrics: + address: "" + level: None + readers: [] + resource: {} + traces: + processors: [] + propagators: [] \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/base_apm_config.conf b/translator/tocwconfig/sampleConfig/base_apm_config.conf new file mode 100644 index 0000000000..007bb60efb --- /dev/null +++ b/translator/tocwconfig/sampleConfig/base_apm_config.conf @@ -0,0 +1,27 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "host_name_from_env" + interval = "60s" + logfile = "" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + +[outputs] + + [[outputs.cloudwatchlogs]] + endpoint_override = "https://fake_endpoint" + force_flush_interval = "5s" + log_stream_name = "host_name_from_env" + region = "us-east-1" + +[processors] diff --git a/translator/tocwconfig/sampleConfig/base_apm_config.json b/translator/tocwconfig/sampleConfig/base_apm_config.json new file mode 100644 index 0000000000..2d4daf5fe2 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/base_apm_config.json @@ -0,0 +1,17 @@ +{ + "agent": { + "region": "us-east-1" + }, + "logs": { + "metrics_collected": { + "apm": {} + }, + "endpoint_override":"https://fake_endpoint" + }, + "traces": { + "traces_collected": { + "apm": {} + }, + "endpoint_override":"https://fake_endpoint" + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/base_apm_config.yaml b/translator/tocwconfig/sampleConfig/base_apm_config.yaml new file mode 100644 index 0000000000..4f04d82f72 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/base_apm_config.yaml @@ -0,0 +1,490 @@ +connectors: {} +exporters: + awsemf/apm: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/apm/eks + log_retention: 0 + log_stream_name: "" + max_retries: 2 + metric_declarations: + - dimensions: + - - EKS.Cluster + - K8s.Namespace + - Operation + - Service + - - EKS.Cluster + - K8s.Namespace + - Service + label_matchers: + - label_names: + - aws.span.kind + regex: ^(SERVER|CONSUMER|LOCAL_ROOT)$ + separator: ; + metric_name_selectors: + - Latency + - Fault + - Error + - dimensions: + - - EKS.Cluster + - K8s.Namespace + - K8s.RemoteNamespace + - Operation + - RemoteOperation + - RemoteService + - RemoteTarget + - Service + - - EKS.Cluster + - K8s.Namespace + - K8s.RemoteNamespace + - Operation + - RemoteOperation + - RemoteService + - Service + - - EKS.Cluster + - K8s.Namespace + - Operation + - RemoteOperation + - RemoteService + - RemoteTarget + - Service + - - EKS.Cluster + - K8s.Namespace + - Operation + - RemoteOperation + - RemoteService + - Service + - - EKS.Cluster + - K8s.Namespace + - K8s.RemoteNamespace + - RemoteService + - Service + - - EKS.Cluster + - K8s.Namespace + - RemoteService + - Service + - - EKS.Cluster + - K8s.Namespace + - K8s.RemoteNamespace + - RemoteOperation + - RemoteService + - RemoteTarget + - Service + - - EKS.Cluster + - K8s.Namespace + - K8s.RemoteNamespace + - RemoteOperation + - RemoteService + - Service + - - EKS.Cluster + - K8s.Namespace + - RemoteOperation + - RemoteService + - RemoteTarget + - Service + - - EKS.Cluster + - K8s.Namespace + - RemoteOperation + - RemoteService + - Service + label_matchers: + - label_names: + - aws.span.kind + regex: ^(CLIENT|PRODUCER)$ + separator: ; + metric_name_selectors: + - Latency + - Fault + - Error + metric_descriptors: [] + namespace: AWS/APM + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: [] + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: false + retain_initial_value_of_delta_metric: false + role_arn: "" + shared_credentials_file: [] + version: "1" + awsxray/apm: + aws_log_groups: [] + certificate_file_path: "" + endpoint: https://fake_endpoint + imds_retries: 1 + index_all_attributes: false + indexed_attributes: + - aws.local.service + - aws.local.operation + - aws.remote.service + - aws.remote.operation + - EKS.Cluster + - K8s.Namespace + - K8s.RemoteNamespace + - aws.remote.target + local_mode: false + max_retries: 2 + no_verify_ssl: false + num_workers: 8 + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + role_arn: "" + shared_credentials_file: [] + telemetry: + enabled: true + include_metadata: true +extensions: + awsproxy/apm: + aws_endpoint: "" + endpoint: 0.0.0.0:2000 + local_mode: false + proxy_address: "" + region: "" + role_arn: "" +processors: + awsapm: + resolvers: + - eks + rules: [] + resourcedetection: + aks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + attributes: [] + auth: null + azure: + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + compression: "" + consul: + address: "" + datacenter: "" + meta: {} + namespace: "" + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + token: '[REDACTED]' + token_file: "" + detectors: + - eks + - env + - ec2 + docker: + resource_attributes: + host.name: + enabled: true + os.type: + enabled: true + ec2: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.image.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + tags: + - ^kubernetes.io/cluster/.*$ + ecs: + resource_attributes: + aws.ecs.cluster.arn: + enabled: true + aws.ecs.launchtype: + enabled: true + aws.ecs.task.arn: + enabled: true + aws.ecs.task.family: + enabled: true + aws.ecs.task.revision: + enabled: true + aws.log.group.arns: + enabled: true + aws.log.group.names: + enabled: true + aws.log.stream.arns: + enabled: true + aws.log.stream.names: + enabled: true + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + eks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + elasticbeanstalk: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + deployment.environment: + enabled: true + service.instance.id: + enabled: true + service.version: + enabled: true + endpoint: "" + gcp: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.id: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + gcp.cloud_run.job.execution: + enabled: true + gcp.cloud_run.job.task_index: + enabled: true + gcp.gce.instance.hostname: + enabled: false + gcp.gce.instance.name: + enabled: false + host.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + k8s.cluster.name: + enabled: true + headers: {} + heroku: + resource_attributes: + cloud.provider: + enabled: true + heroku.app.id: + enabled: true + heroku.dyno.id: + enabled: true + heroku.release.commit: + enabled: true + heroku.release.creation_timestamp: + enabled: true + service.instance.id: + enabled: true + service.name: + enabled: true + service.version: + enabled: true + idle_conn_timeout: 1m30s + lambda: + resource_attributes: + aws.log.group.names: + enabled: true + aws.log.stream.names: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.instance: + enabled: true + faas.max_memory: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + max_conns_per_host: null + max_idle_conns: 100 + max_idle_conns_per_host: null + openshift: + address: "" + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + k8s.cluster.name: + enabled: true + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + token: "" + override: false + read_buffer_size: 0 + system: + hostname_sources: [] + resource_attributes: + host.arch: + enabled: false + host.id: + enabled: false + host.name: + enabled: true + os.description: + enabled: false + os.type: + enabled: true + timeout: 2s + write_buffer_size: 0 +receivers: + otlp/apm: + protocols: + grpc: + auth: null + endpoint: 0.0.0.0:4317 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp + write_buffer_size: 0 + http: + auth: null + cors: null + endpoint: 0.0.0.0:4318 + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + response_headers: {} + tls: null + traces_url_path: /v1/traces +service: + extensions: + - awsproxy/apm + pipelines: + metrics/apm: + exporters: + - awsemf/apm + processors: + - resourcedetection + - awsapm + receivers: + - otlp/apm + traces/apm: + exporters: + - awsxray/apm + processors: + - resourcedetection + - awsapm + receivers: + - otlp/apm + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: [] + initial_fields: {} + level: info + output_paths: [] + sampling: + initial: 2 + thereafter: 500 + metrics: + address: "" + level: None + readers: [] + resource: {} + traces: + processors: [] + propagators: [] \ No newline at end of file diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index fffa845d8c..a34ab0d290 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -54,7 +54,26 @@ func TestBaseContainerInsightsConfig(t *testing.T) { t.Setenv(config.HOST_IP, "127.0.0.1") expectedEnvVars := map[string]string{} checkTranslation(t, "base_container_insights_config", "linux", expectedEnvVars, "") - checkTranslation(t, "base_container_insights_config", "darwin", nil, "") +} + +func TestBaseAPMConfig(t *testing.T) { + resetContext(t) + context.CurrentContext().SetRunInContainer(true) + t.Setenv(config.HOST_NAME, "host_name_from_env") + t.Setenv(config.HOST_IP, "127.0.0.1") + expectedEnvVars := map[string]string{} + checkTranslation(t, "base_apm_config", "linux", expectedEnvVars, "") + checkTranslation(t, "base_apm_config", "windows", expectedEnvVars, "") +} + +func TestAPMAndKubernetesConfig(t *testing.T) { + resetContext(t) + context.CurrentContext().SetRunInContainer(true) + t.Setenv(config.HOST_NAME, "host_name_from_env") + t.Setenv(config.HOST_IP, "127.0.0.1") + expectedEnvVars := map[string]string{} + checkTranslation(t, "apm_and_kubernetes_config", "linux", expectedEnvVars, "") + checkTranslation(t, "apm_and_kubernetes_config", "windows", expectedEnvVars, "") } func TestEmfAndKubernetesConfig(t *testing.T) { @@ -65,7 +84,6 @@ func TestEmfAndKubernetesConfig(t *testing.T) { t.Setenv(config.HOST_IP, "127.0.0.1") expectedEnvVars := map[string]string{} checkTranslation(t, "emf_and_kubernetes_config", "linux", expectedEnvVars, "") - checkTranslation(t, "emf_and_kubernetes_config", "darwin", nil, "") } func TestKubernetesModeOnPremiseConfig(t *testing.T) { diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index be3f9d2190..2de6bb471b 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -13,6 +13,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "gopkg.in/yaml.v3" ) const ( @@ -61,6 +62,19 @@ const ( PipelineNameHost = "host" PipelineNameHostDeltaMetrics = "hostDeltaMetrics" PipelineNameEmfLogs = "emf_logs" + APM = "apm" + + APMRules = "rules" +) + +var ( + APMTraces = ConfigKey(TracesKey, TracesCollectedKey, APM) + APMMetrics = ConfigKey(LogsKey, MetricsCollectedKey, APM) + + APMConfigKeys = map[component.DataType]string{ + component.DataTypeTraces: APMTraces, + component.DataTypeMetrics: APMMetrics, + } ) // Translator is used to translate the JSON config into an @@ -306,3 +320,16 @@ func GetOrDefaultDuration(conf *confmap.Conf, keychain []string, defaultDuration } return defaultDuration } + +func GetYamlFileToYamlConfig(cfg interface{}, yamlFile string) (interface{}, error) { + var cfgMap map[string]interface{} + if err := yaml.Unmarshal([]byte(yamlFile), &cfgMap); err != nil { + return nil, fmt.Errorf("unable to read default config: %w", err) + } + + conf := confmap.NewFromStringMap(cfgMap) + if err := conf.Unmarshal(&cfg); err != nil { + return nil, fmt.Errorf("unable to unmarshal config: %w", err) + } + return cfg, nil +} diff --git a/translator/translate/otel/exporter/awsemf/awsemf_default_apm.yaml b/translator/translate/otel/exporter/awsemf/awsemf_default_apm.yaml new file mode 100644 index 0000000000..d5256187c6 --- /dev/null +++ b/translator/translate/otel/exporter/awsemf/awsemf_default_apm.yaml @@ -0,0 +1,34 @@ +log_group_name: "/aws/apm/eks" +namespace: "AWS/APM" +dimension_rollup_option: "NoDimensionRollup" +metric_declarations: + - dimensions: + - [EKS.Cluster, K8s.Namespace, Service, Operation] + - [EKS.Cluster, K8s.Namespace, Service] + label_matchers: + - label_names: + - aws.span.kind + regex: '^(SERVER|CONSUMER|LOCAL_ROOT)$' + metric_name_selectors: + - Latency + - Fault + - Error + - dimensions: + - [EKS.Cluster, K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] + - [EKS.Cluster, K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace] + - [EKS.Cluster, K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] + - [EKS.Cluster, K8s.Namespace, Service, Operation, RemoteService, RemoteOperation] + - [EKS.Cluster, K8s.Namespace, Service, RemoteService, K8s.RemoteNamespace] + - [EKS.Cluster, K8s.Namespace, Service, RemoteService] + - [EKS.Cluster, K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] + - [EKS.Cluster, K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace] + - [EKS.Cluster, K8s.Namespace, Service, RemoteService, RemoteOperation, RemoteTarget] + - [EKS.Cluster, K8s.Namespace, Service, RemoteService, RemoteOperation] + label_matchers: + - label_names: + - aws.span.kind + regex: '^(CLIENT|PRODUCER)$' + metric_name_selectors: + - Latency + - Fault + - Error \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 0eee896963..962490e101 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -28,6 +28,9 @@ var defaultKubernetesConfig string //go:embed awsemf_default_prometheus.yaml var defaultPrometheusConfig string +//go:embed awsemf_default_apm.yaml +var defaultAPMConfig string + var ( ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) kubernetesBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey) @@ -65,6 +68,8 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { defaultConfig = defaultKubernetesConfig } else if isPrometheus(c) { defaultConfig = defaultPrometheusConfig + } else if isAPM(c) { + defaultConfig = defaultAPMConfig } else { return cfg, nil } @@ -104,6 +109,10 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { return cfg, nil } +func isAPM(conf *confmap.Conf) bool { + return conf.IsSet(common.APMMetrics) +} + func isEcs(conf *confmap.Conf) bool { return conf.IsSet(ecsBasePathKey) } diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index b6243ffb6a..b93ac3272a 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -583,6 +583,65 @@ func TestTranslator(t *testing.T) { "metric_descriptors": nilMetricDescriptorsSlice, }, }, + "WithAPMEnabled": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }}, + want: map[string]interface{}{ + "namespace": "AWS/APM", + "log_group_name": "/aws/apm/eks", + "log_stream_name": "", + "dimension_rollup_option": "NoDimensionRollup", + "disable_metric_extraction": false, + "enhanced_container_insights": false, + "parse_json_encoded_attr_values": nilSlice, + "output_destination": "cloudwatch", + "eks_fargate_container_insights_enabled": false, + "resource_to_telemetry_conversion": resourcetotelemetry.Settings{ + Enabled: false, + }, + "metric_declarations": []*awsemfexporter.MetricDeclaration{ + { + Dimensions: [][]string{ + {"EKS.Cluster", "K8s.Namespace", "Service", "Operation"}, + {"EKS.Cluster", "K8s.Namespace", "Service"}, + }, + LabelMatchers: []*awsemfexporter.LabelMatcher{ + { + LabelNames: []string{"aws.span.kind"}, + Regex: "^(SERVER|CONSUMER|LOCAL_ROOT)$", + }, + }, + MetricNameSelectors: []string{"Latency", "Fault", "Error"}, + }, + { + Dimensions: [][]string{ + {"EKS.Cluster", "K8s.Namespace", "Service", "Operation", "RemoteService", "RemoteOperation", "K8s.RemoteNamespace", "RemoteTarget"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "Operation", "RemoteService", "RemoteOperation", "K8s.RemoteNamespace"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "Operation", "RemoteService", "RemoteOperation", "RemoteTarget"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "Operation", "RemoteService", "RemoteOperation"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "K8s.RemoteNamespace"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "RemoteOperation", "K8s.RemoteNamespace", "RemoteTarget"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "RemoteOperation", "K8s.RemoteNamespace"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "RemoteOperation", "RemoteTarget"}, + {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "RemoteOperation"}, + }, + LabelMatchers: []*awsemfexporter.LabelMatcher{ + { + LabelNames: []string{"aws.span.kind"}, + Regex: "^(CLIENT|PRODUCER)$", + }, + }, + MetricNameSelectors: []string{"Latency", "Fault", "Error"}, + }, + }, + "metric_descriptors": nilMetricDescriptorsSlice, + }, + }, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { diff --git a/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml b/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml new file mode 100644 index 0000000000..7480bd4e2e --- /dev/null +++ b/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml @@ -0,0 +1,2 @@ +# Add new attributes as annotations for diagnostics. +indexed_attributes: [aws.local.service, aws.local.operation, aws.remote.service, aws.remote.operation, EKS.Cluster, K8s.Namespace, K8s.RemoteNamespace, aws.remote.target] \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsxray/translator.go b/translator/translate/otel/exporter/awsxray/translator.go index 1b1eeeea35..06b1869e4c 100644 --- a/translator/translate/otel/exporter/awsxray/translator.go +++ b/translator/translate/otel/exporter/awsxray/translator.go @@ -4,6 +4,7 @@ package awsxray import ( + _ "embed" "fmt" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter" @@ -21,6 +22,9 @@ const ( resourceARNKey = "resource_arn" ) +//go:embed awsxray_default_apm.yaml +var defaultAPMConfig string + type translator struct { name string factory exporter.Factory @@ -48,6 +52,14 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.TracesKey} } cfg := t.factory.CreateDefaultConfig().(*awsxrayexporter.Config) + + if isAPM(conf) { + cfg.IndexedAttributes = []string{ + "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", + "EKS.Cluster", "K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", + } + } + c := confmap.NewFromStringMap(map[string]interface{}{ "telemetry": map[string]interface{}{ "enabled": true, @@ -104,3 +116,7 @@ func getRegion(conf *confmap.Conf) string { } return region } + +func isAPM(conf *confmap.Conf) bool { + return conf.IsSet(common.APMTraces) +} diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index 848c3d12bc..1ff05def06 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -51,6 +51,33 @@ func TestTranslator(t *testing.T) { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), }, + "WithAPMEnabled": { + input: map[string]interface{}{ + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }}, + want: confmap.NewFromStringMap(map[string]interface{}{ + "indexed_attributes": []string{ + "aws.local.service", + "aws.local.operation", + "aws.remote.service", + "aws.remote.operation", + "EKS.Cluster", + "K8s.Namespace", + "K8s.RemoteNamespace", + "aws.remote.target", + }, + "region": "us-east-1", + "role_arn": "global_arn", + "imds_retries": 1, + "telemetry": map[string]interface{}{ + "enabled": true, + "include_metadata": true, + }, + }), + }, } factory := awsxrayexporter.NewFactory() for name, testCase := range testCases { diff --git a/translator/translate/otel/extension/awsproxy/translator.go b/translator/translate/otel/extension/awsproxy/translator.go new file mode 100644 index 0000000000..4aeab63887 --- /dev/null +++ b/translator/translate/otel/extension/awsproxy/translator.go @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsproxy + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/extension" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +type translator struct { + name string + factory extension.Factory +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslator() common.Translator[component.Config] { + return NewTranslatorWithName("") +} + +func NewTranslatorWithName(name string) common.Translator[component.Config] { + return &translator{name, awsproxy.NewFactory()} +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*awsproxy.Config) + return cfg, nil +} diff --git a/translator/translate/otel/extension/awsproxy/translator_test.go b/translator/translate/otel/extension/awsproxy/translator_test.go new file mode 100644 index 0000000000..5934a92e91 --- /dev/null +++ b/translator/translate/otel/extension/awsproxy/translator_test.go @@ -0,0 +1,26 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsproxy + +import ( + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap" +) + +func TestTranslate(t *testing.T) { + tt := NewTranslator() + conf := confmap.NewFromStringMap(map[string]interface{}{}) + got, err := tt.Translate(conf) + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*awsproxy.Config) + require.True(t, ok) + wantCfg := awsproxy.NewFactory().CreateDefaultConfig() + assert.Equal(t, wantCfg, gotCfg) + } +} diff --git a/translator/translate/otel/pipeline/apm/translator.go b/translator/translate/otel/pipeline/apm/translator.go new file mode 100644 index 0000000000..cc6a40209d --- /dev/null +++ b/translator/translate/otel/pipeline/apm/translator.go @@ -0,0 +1,64 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package apm + +import ( + "fmt" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsxray" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/awsproxy" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/awsapm" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" +) + +const ( + pipelineName = "apm" +) + +type translator struct { + dataType component.DataType +} + +var _ common.Translator[*common.ComponentTranslators] = (*translator)(nil) + +func NewTranslator(dataType component.DataType) common.Translator[*common.ComponentTranslators] { + return &translator{ + dataType, + } +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.dataType, pipelineName) +} + +func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) { + configKey, ok := common.APMConfigKeys[t.dataType] + if !ok { + return nil, fmt.Errorf("no config key defined for data type: %s", t.dataType) + } + if conf == nil || !conf.IsSet(configKey) { + return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: configKey} + } + + translators := &common.ComponentTranslators{ + Receivers: common.NewTranslatorMap(otlp.NewTranslatorWithName(common.APM, otlp.WithDataType(t.dataType))), + Processors: common.NewTranslatorMap(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType)), awsapm.NewTranslator(awsapm.WithDataType(t.dataType))), + Exporters: common.NewTranslatorMap[component.Config](), + Extensions: common.NewTranslatorMap[component.Config](), + } + + if t.dataType == component.DataTypeTraces { + translators.Exporters.Set(awsxray.NewTranslatorWithName(common.APM)) + translators.Extensions.Set(awsproxy.NewTranslatorWithName(common.APM)) + } else { + translators.Exporters.Set(awsemf.NewTranslatorWithName(common.APM)) + } + return translators, nil +} diff --git a/translator/translate/otel/pipeline/apm/translator_test.go b/translator/translate/otel/pipeline/apm/translator_test.go new file mode 100644 index 0000000000..f7ba4e5c28 --- /dev/null +++ b/translator/translate/otel/pipeline/apm/translator_test.go @@ -0,0 +1,117 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package apm + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/internal/util/collections" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +func TestTranslatorTraces(t *testing.T) { + type want struct { + receivers []string + processors []string + exporters []string + extensions []string + } + tt := NewTranslator(component.DataTypeTraces) + assert.EqualValues(t, "traces/apm", tt.ID().String()) + testCases := map[string]struct { + input map[string]interface{} + want *want + wantErr error + }{ + "WithoutTracesCollectedKey": { + input: map[string]interface{}{}, + wantErr: &common.MissingKeyError{ID: tt.ID(), JsonKey: fmt.Sprint(common.APMTraces)}, + }, + "WithAPMEnabledTraces": { + input: map[string]interface{}{ + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }, + }, + want: &want{ + receivers: []string{"otlp/apm"}, + processors: []string{"resourcedetection", "awsapm"}, + exporters: []string{"awsxray/apm"}, + extensions: []string{"awsproxy/apm"}, + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + assert.Equal(t, testCase.wantErr, err) + if testCase.want == nil { + assert.Nil(t, got) + } else { + require.NotNil(t, got) + assert.Equal(t, testCase.want.receivers, collections.MapSlice(got.Receivers.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.processors, collections.MapSlice(got.Processors.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.exporters, collections.MapSlice(got.Exporters.Keys(), component.ID.String)) + } + }) + } +} + +func TestTranslatorMetrics(t *testing.T) { + type want struct { + receivers []string + processors []string + exporters []string + } + tt := NewTranslator(component.DataTypeMetrics) + assert.EqualValues(t, "metrics/apm", tt.ID().String()) + testCases := map[string]struct { + input map[string]interface{} + want *want + wantErr error + }{ + "WithoutMetricsCollectedKey": { + input: map[string]interface{}{}, + wantErr: &common.MissingKeyError{ID: tt.ID(), JsonKey: fmt.Sprint(common.APMMetrics)}, + }, + "WithAPMEnabledMetrics": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }, + }, + want: &want{ + receivers: []string{"otlp/apm"}, + processors: []string{"resourcedetection", "awsapm"}, + exporters: []string{"awsemf/apm"}, + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + assert.Equal(t, testCase.wantErr, err) + if testCase.want == nil { + assert.Nil(t, got) + } else { + require.NotNil(t, got) + assert.Equal(t, testCase.want.receivers, collections.MapSlice(got.Receivers.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.processors, collections.MapSlice(got.Processors.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.exporters, collections.MapSlice(got.Exporters.Keys(), component.ID.String)) + } + }) + } +} diff --git a/translator/translate/otel/processor/awsapm/testdata/config.yaml b/translator/translate/otel/processor/awsapm/testdata/config.yaml new file mode 100644 index 0000000000..06bf8b38d1 --- /dev/null +++ b/translator/translate/otel/processor/awsapm/testdata/config.yaml @@ -0,0 +1 @@ +resolvers: ["eks"] \ No newline at end of file diff --git a/translator/translate/otel/processor/awsapm/testdata/invalidRulesConfig.json b/translator/translate/otel/processor/awsapm/testdata/invalidRulesConfig.json new file mode 100644 index 0000000000..17f9fc537d --- /dev/null +++ b/translator/translate/otel/processor/awsapm/testdata/invalidRulesConfig.json @@ -0,0 +1,20 @@ +{ + "logs": { + "metrics_collected": { + "apm": { + "rules": [ + { + "selectors": [ + { + "dimension": "dimension1", + "match": "match1" + } + ], + "action": "replace", + "rule_name": "replace01" + } + ] + } + } + } +} diff --git a/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.json b/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.json new file mode 100644 index 0000000000..bf3c011720 --- /dev/null +++ b/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.json @@ -0,0 +1,63 @@ +{ + "logs": { + "metrics_collected": { + "apm": { + "rules": [ + { + "selectors": [ + { + "dimension": "Operation", + "match": "POST *" + }, + { + "dimension": "RemoteService", + "match": "*" + } + ], + "action": "keep" + }, + { + "selectors": [ + { + "dimension": "Operation", + "match": "GET *" + }, + { + "dimension": "RemoteService", + "match": "*" + } + ], + "action": "keep", + "rule_name": "keep02" + }, + { + "selectors": [ + { + "dimension": "Operation", + "match": "POST *" + } + ], + "action": "drop", + "rule_name": "drop01" + }, + { + "selectors": [ + { + "dimension": "Operation", + "match": "*" + } + ], + "replacements": [ + { + "target_dimension": "RemoteOperation", + "value": "This is a test string" + } + ], + "action": "replace", + "rule_name": "replace01" + } + ] + } + } + } +} \ No newline at end of file diff --git a/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.yaml b/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.yaml new file mode 100644 index 0000000000..0d69e235f6 --- /dev/null +++ b/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.yaml @@ -0,0 +1,28 @@ +resolvers: ["eks"] +rules: + - selectors: + - dimension: Operation + match: "POST *" + - dimension: RemoteService + match: "*" + action: keep + - selectors: + - dimension: Operation + match: "GET *" + - dimension: RemoteService + match: "*" + action: keep + rule_name: "keep02" + - selectors: + - dimension: Operation + match: "POST *" + action: drop + rule_name: "drop01" + - selectors: + - dimension: Operation + match: "*" + replacements: + - target_dimension: RemoteOperation + value: "This is a test string" + action: replace + rule_name: "replace01" \ No newline at end of file diff --git a/translator/translate/otel/processor/awsapm/translator.go b/translator/translate/otel/processor/awsapm/translator.go new file mode 100644 index 0000000000..3d5bed51c9 --- /dev/null +++ b/translator/translate/otel/processor/awsapm/translator.go @@ -0,0 +1,119 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsapm + +import ( + _ "embed" + "errors" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/processor" + + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor" + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +type translator struct { + name string + dataType component.DataType + factory processor.Factory +} + +type Option interface { + apply(t *translator) +} + +type optionFunc func(t *translator) + +func (o optionFunc) apply(t *translator) { + o(t) +} + +// WithDataType determines where the translator should look to find +// the configuration. +func WithDataType(dataType component.DataType) Option { + return optionFunc(func(t *translator) { + t.dataType = dataType + }) +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslator(opts ...Option) common.Translator[component.Config] { + t := &translator{factory: awsapmprocessor.NewFactory()} + for _, opt := range opts { + opt.apply(t) + } + return t +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + configKey := common.APMConfigKeys[t.dataType] + cfg := t.factory.CreateDefaultConfig().(*awsapmprocessor.Config) + cfg.Resolvers = []string{"eks"} + return t.translateCustomRules(conf, configKey, cfg) +} + +func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, cfg *awsapmprocessor.Config) (component.Config, error) { + var rules []customconfiguration.Rule + rulesConfigKey := common.ConfigKey(configKey, common.APMRules) + if conf.IsSet(rulesConfigKey) { + for _, rule := range conf.Get(rulesConfigKey).([]interface{}) { + ruleConfig := customconfiguration.Rule{} + ruleMap := rule.(map[string]interface{}) + selectors := ruleMap["selectors"].([]interface{}) + action := ruleMap["action"].(string) + + ruleConfig.Selectors = getServiceSelectors(selectors) + if ruleName, ok := ruleMap["rule_name"]; ok { + ruleConfig.RuleName = ruleName.(string) + } + ruleConfig.Action = action + if action == "replace" { + replacements, ok := ruleMap["replacements"] + if !ok { + return nil, errors.New("replace action set, but no replacements defined for service rule") + } + ruleConfig.Replacements = getServiceReplacements(replacements) + } + + rules = append(rules, ruleConfig) + } + cfg.Rules = rules + } + + return cfg, nil +} + +func getServiceSelectors(selectorsList []interface{}) []customconfiguration.Selector { + var selectors []customconfiguration.Selector + for _, selector := range selectorsList { + selectorConfig := customconfiguration.Selector{} + selectorsMap := selector.(map[string]interface{}) + + selectorConfig.Dimension = selectorsMap["dimension"].(string) + selectorConfig.Match = selectorsMap["match"].(string) + selectors = append(selectors, selectorConfig) + } + return selectors +} + +func getServiceReplacements(replacementsList interface{}) []customconfiguration.Replacement { + var replacements []customconfiguration.Replacement + for _, replacement := range replacementsList.([]interface{}) { + replacementConfig := customconfiguration.Replacement{} + replacementMap := replacement.(map[string]interface{}) + + replacementConfig.TargetDimension = replacementMap["target_dimension"].(string) + replacementConfig.Value = replacementMap["value"].(string) + replacements = append(replacements, replacementConfig) + } + return replacements +} diff --git a/translator/translate/otel/processor/awsapm/translator_test.go b/translator/translate/otel/processor/awsapm/translator_test.go new file mode 100644 index 0000000000..7f7303c8a8 --- /dev/null +++ b/translator/translate/otel/processor/awsapm/translator_test.go @@ -0,0 +1,80 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package awsapm + +import ( + _ "embed" + "encoding/json" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +var ( + //go:embed testdata/config.yaml + validAPMYaml string + //go:embed testdata/validRulesConfig.json + validAPMRulesConfig string + //go:embed testdata/validRulesConfig.yaml + validAPMRulesYaml string + //go:embed testdata/invalidRulesConfig.json + invalidAPMRulesConfig string +) + +func TestTranslate(t *testing.T) { + var validJsonMap, invalidJsonMap map[string]interface{} + json.Unmarshal([]byte(validAPMRulesConfig), &validJsonMap) + json.Unmarshal([]byte(invalidAPMRulesConfig), &invalidJsonMap) + + tt := NewTranslator(WithDataType(component.DataTypeMetrics)) + testCases := map[string]struct { + input map[string]interface{} + want string + wantErr error + }{ + //The config for the awsapm processor is https://code.amazon.com/packages/AWSTracingSamplePetClinic/blobs/97ce3c409986ac8ae014de1e3fe71fdb98080f22/--/eks/apm/auto-instrumentation-new.yaml#L20 + //The awsapm processor config does not have a platform field, instead it gets added to resolvers when marshalled + "WithAPMEnabled": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }}, + want: validAPMYaml, + }, + "WithAPMCustomRulesEnabled": { + input: validJsonMap, + want: validAPMRulesYaml, + }, + "WithInvalidAPMCustomRulesEnabled": { + input: invalidJsonMap, + wantErr: errors.New("replace action set, but no replacements defined for service rule"), + }, + } + factory := awsapmprocessor.NewFactory() + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + assert.Equal(t, testCase.wantErr, err) + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*awsapmprocessor.Config) + require.True(t, ok) + wantCfg := factory.CreateDefaultConfig() + yamlConfig, err := common.GetYamlFileToYamlConfig(wantCfg, testCase.want) + require.NoError(t, err) + assert.Equal(t, yamlConfig.(*awsapmprocessor.Config), gotCfg) + } + }) + } +} diff --git a/translator/translate/otel/processor/resourcedetection/configs/config.yaml b/translator/translate/otel/processor/resourcedetection/configs/config.yaml new file mode 100644 index 0000000000..810aceb6a8 --- /dev/null +++ b/translator/translate/otel/processor/resourcedetection/configs/config.yaml @@ -0,0 +1,6 @@ +detectors: [eks, env, ec2] +override: false +timeout: 2s +ec2: + tags: + - ^kubernetes.io/cluster/.*$ \ No newline at end of file diff --git a/translator/translate/otel/processor/resourcedetection/translator.go b/translator/translate/otel/processor/resourcedetection/translator.go new file mode 100644 index 0000000000..485eb92020 --- /dev/null +++ b/translator/translate/otel/processor/resourcedetection/translator.go @@ -0,0 +1,61 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resourcedetection + +import ( + _ "embed" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/processor" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +//go:embed configs/config.yaml +var apmAwsResourceDetectionConfig string + +type translator struct { + name string + dataType component.DataType + factory processor.Factory +} + +type Option interface { + apply(t *translator) +} + +type optionFunc func(t *translator) + +func (o optionFunc) apply(t *translator) { + o(t) +} + +// WithDataType determines where the translator should look to find +// the configuration. +func WithDataType(dataType component.DataType) Option { + return optionFunc(func(t *translator) { + t.dataType = dataType + }) +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslator(opts ...Option) common.Translator[component.Config] { + t := &translator{factory: resourcedetectionprocessor.NewFactory()} + for _, opt := range opts { + opt.apply(t) + } + return t +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*resourcedetectionprocessor.Config) + return common.GetYamlFileToYamlConfig(cfg, apmAwsResourceDetectionConfig) +} diff --git a/translator/translate/otel/processor/resourcedetection/translator_test.go b/translator/translate/otel/processor/resourcedetection/translator_test.go new file mode 100644 index 0000000000..faf784c481 --- /dev/null +++ b/translator/translate/otel/processor/resourcedetection/translator_test.go @@ -0,0 +1,60 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resourcedetection + +import ( + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" +) + +func TestTranslate(t *testing.T) { + tt := NewTranslator(WithDataType(component.DataTypeTraces)) + testCases := map[string]struct { + input map[string]interface{} + want *confmap.Conf + wantErr error + }{ + "WithAPMEnabled": { + input: map[string]interface{}{ + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }}, + want: confmap.NewFromStringMap(map[string]interface{}{ + "detectors": []interface{}{ + "eks", + "env", + "ec2", + }, + "timeout": "2s", + "override": false, + "ec2": map[string]interface{}{ + "tags": []interface{}{"^kubernetes.io/cluster/.*$"}, + }, + }), + }, + } + factory := resourcedetectionprocessor.NewFactory() + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + assert.Equal(t, testCase.wantErr, err) + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*resourcedetectionprocessor.Config) + require.True(t, ok) + wantCfg := factory.CreateDefaultConfig() + require.NoError(t, component.UnmarshalConfig(testCase.want, wantCfg)) + assert.Equal(t, wantCfg, gotCfg) + } + }) + } +} diff --git a/translator/translate/otel/receiver/otlp/apm_config.yaml b/translator/translate/otel/receiver/otlp/apm_config.yaml new file mode 100644 index 0000000000..c14c705d9d --- /dev/null +++ b/translator/translate/otel/receiver/otlp/apm_config.yaml @@ -0,0 +1,5 @@ +protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 \ No newline at end of file diff --git a/translator/translate/otel/receiver/otlp/translator.go b/translator/translate/otel/receiver/otlp/translator.go index 01e57e3fa7..4573d017b8 100644 --- a/translator/translate/otel/receiver/otlp/translator.go +++ b/translator/translate/otel/receiver/otlp/translator.go @@ -4,6 +4,7 @@ package otlp import ( + _ "embed" "fmt" "go.opentelemetry.io/collector/component" @@ -23,6 +24,9 @@ var ( configKeys = map[component.DataType]string{ component.DataTypeTraces: common.ConfigKey(common.TracesKey, common.TracesCollectedKey, common.OtlpKey), } + + //go:embed apm_config.yaml + apmConfig string ) type translator struct { @@ -59,11 +63,26 @@ func NewTranslator(opts ...Option) common.Translator[component.Config] { return t } +func NewTranslatorWithName(name string, opts ...Option) common.Translator[component.Config] { + t := &translator{name: name, factory: otlpreceiver.NewFactory()} + for _, opt := range opts { + opt.apply(t) + } + return t +} + func (t *translator) ID() component.ID { return component.NewIDWithName(t.factory.Type(), t.name) } func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*otlpreceiver.Config) + + // TODO: Should follow pattern done in awsemf and awsexray exporter translations (i.e should be integrated with standard otlp translation) + if t.name == common.APM { + return common.GetYamlFileToYamlConfig(cfg, apmConfig) + } + configKey, ok := configKeys[t.dataType] if !ok { return nil, fmt.Errorf("no config key defined for data type: %s", t.dataType) @@ -71,7 +90,6 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { if conf == nil || !conf.IsSet(configKey) { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: configKey} } - cfg := t.factory.CreateDefaultConfig().(*otlpreceiver.Config) cfg.GRPC.NetAddr.Endpoint = defaultGrpcEndpoint cfg.HTTP.Endpoint = defaultHttpEndpoint if endpoint, ok := common.GetString(conf, common.ConfigKey(configKey, "grpc_endpoint")); ok { diff --git a/translator/translate/otel/receiver/otlp/translator_test.go b/translator/translate/otel/receiver/otlp/translator_test.go index 3eb8d89c23..42049ebe64 100644 --- a/translator/translate/otel/receiver/otlp/translator_test.go +++ b/translator/translate/otel/receiver/otlp/translator_test.go @@ -74,3 +74,47 @@ func TestTracesTranslator(t *testing.T) { }) } } + +func TestTranslateAPM(t *testing.T) { + tt := NewTranslatorWithName(common.APM, WithDataType(component.DataTypeTraces)) + testCases := map[string]struct { + input map[string]interface{} + want *confmap.Conf + wantErr error + }{ + "WithAPMEnabledTraces": { + input: map[string]interface{}{ + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }}, + want: confmap.NewFromStringMap(map[string]interface{}{ + "protocols": map[string]interface{}{ + "grpc": map[string]interface{}{ + "endpoint": "0.0.0.0:4317", + }, + "http": map[string]interface{}{ + "endpoint": "0.0.0.0:4318", + }, + }, + }), + }, + } + factory := otlpreceiver.NewFactory() + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + assert.Equal(t, testCase.wantErr, err) + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*otlpreceiver.Config) + require.True(t, ok) + wantCfg := factory.CreateDefaultConfig() + require.NoError(t, component.UnmarshalConfig(testCase.want, wantCfg)) + assert.Equal(t, wantCfg, gotCfg) + } + }) + } +} diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 979d128ec3..4dd8a02f9a 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -21,6 +21,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/apm" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsights" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/emf_logs" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/host" @@ -66,6 +67,8 @@ func Translate(jsonConfig interface{}, os string) (*otelcol.Config, error) { }) translators := common.NewTranslatorMap( + apm.NewTranslator(component.DataTypeTraces), + apm.NewTranslator(component.DataTypeMetrics), host.NewTranslator(common.PipelineNameHost, hostReceivers), host.NewTranslator(common.PipelineNameHostDeltaMetrics, deltaMetricsReceivers), containerinsights.NewTranslator(), diff --git a/translator/translate/otel/translate_otel_test.go b/translator/translate/otel/translate_otel_test.go index c8f4eb85b3..d0e48e4146 100644 --- a/translator/translate/otel/translate_otel_test.go +++ b/translator/translate/otel/translate_otel_test.go @@ -46,6 +46,38 @@ func TestTranslator(t *testing.T) { }, }, }, + "WithAPMMetricsEnabled": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }, + }, + }, + "WithAPMTracesEnabled": { + input: map[string]interface{}{ + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }, + }, + }, + "WithAPMMetricsAndTracesEnabled": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }, + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }, + }, + }, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { From 88da6a6268d711f36375a4edb9cb67cb653b86dc Mon Sep 17 00:00:00 2001 From: Hyunsoo Kim <884273+movence@users.noreply.github.com> Date: Wed, 11 Oct 2023 14:17:38 -0400 Subject: [PATCH 04/38] Fix bug with trace origin value for apm (#552) --- .../tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml | 2 +- translator/tocwconfig/sampleConfig/base_apm_config.yaml | 2 +- .../otel/processor/resourcedetection/configs/config.yaml | 2 +- .../otel/processor/resourcedetection/translator_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml index 1f01de5964..11b1190477 100644 --- a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml @@ -529,7 +529,7 @@ processors: reload_interval: 0s server_name_override: "" token: "" - override: false + override: true read_buffer_size: 0 system: hostname_sources: [] diff --git a/translator/tocwconfig/sampleConfig/base_apm_config.yaml b/translator/tocwconfig/sampleConfig/base_apm_config.yaml index 4f04d82f72..6c620f5eb7 100644 --- a/translator/tocwconfig/sampleConfig/base_apm_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_apm_config.yaml @@ -405,7 +405,7 @@ processors: reload_interval: 0s server_name_override: "" token: "" - override: false + override: true read_buffer_size: 0 system: hostname_sources: [] diff --git a/translator/translate/otel/processor/resourcedetection/configs/config.yaml b/translator/translate/otel/processor/resourcedetection/configs/config.yaml index 810aceb6a8..b957ade437 100644 --- a/translator/translate/otel/processor/resourcedetection/configs/config.yaml +++ b/translator/translate/otel/processor/resourcedetection/configs/config.yaml @@ -1,5 +1,5 @@ detectors: [eks, env, ec2] -override: false +override: true timeout: 2s ec2: tags: diff --git a/translator/translate/otel/processor/resourcedetection/translator_test.go b/translator/translate/otel/processor/resourcedetection/translator_test.go index faf784c481..f022ce2f2e 100644 --- a/translator/translate/otel/processor/resourcedetection/translator_test.go +++ b/translator/translate/otel/processor/resourcedetection/translator_test.go @@ -34,7 +34,7 @@ func TestTranslate(t *testing.T) { "ec2", }, "timeout": "2s", - "override": false, + "override": true, "ec2": map[string]interface{}{ "tags": []interface{}{"^kubernetes.io/cluster/.*$"}, }, From 3da60ca56f2e5fb8ea54794d08394e8aaeab40a3 Mon Sep 17 00:00:00 2001 From: Mahad Janjua <134644284+majanjua-amzn@users.noreply.github.com> Date: Tue, 17 Oct 2023 08:09:37 -0700 Subject: [PATCH 05/38] E2E Testing: Run E2E tests on main-apm (#577) --- .github/workflows/apm-beta-pre-release.yml | 12 + .github/workflows/apm-e2e-test.yml | 244 +++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 .github/workflows/apm-e2e-test.yml diff --git a/.github/workflows/apm-beta-pre-release.yml b/.github/workflows/apm-beta-pre-release.yml index 1888325a11..ca201d50b6 100644 --- a/.github/workflows/apm-beta-pre-release.yml +++ b/.github/workflows/apm-beta-pre-release.yml @@ -17,3 +17,15 @@ jobs: ContainerRepositoryNameAndTag: "apm-beta-pre-release:latest" BucketKey: "apm-beta-pre-release" PackageBucketKey: "apm-beta-pre-release" + + e2e-test: + needs: BuildAndUpload + uses: ./.github/workflows/apm-e2e-test.yml + secrets: inherit + concurrency: + group: 'pulse-cw-agent-test' + cancel-in-progress: false + with: + test-cluster-name: 'pulse-cw-agent-test' + # The following needs to be updated once we go public + apm-cwagent-image-name: '506463145083.dkr.ecr.us-west-2.amazonaws.com/apm-beta-pre-release:latest' \ No newline at end of file diff --git a/.github/workflows/apm-e2e-test.yml b/.github/workflows/apm-e2e-test.yml new file mode 100644 index 0000000000..52d89ee8b1 --- /dev/null +++ b/.github/workflows/apm-e2e-test.yml @@ -0,0 +1,244 @@ +# This is a reusable workflow for running the E2E test for APM. +# It is meant to be called from another workflow. +# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview +name: APM Enablement E2E Testing +on: + workflow_call: + inputs: + test-cluster-name: + required: true + type: string + apm-cwagent-image-name: + required: true + type: string + +permissions: + id-token: write + contents: read + +env: + AWS_DEFAULT_REGION: us-east-1 + SAMPLE_APP_NAMESPACE: sample-app-namespace + METRIC_NAMESPACE: AWS/APM + APM_LOG_GROUP: /aws/apm/eks + +jobs: + apm-e2e-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + # Checkout apm branch and get only the required resources for testing + ref: apm + sparse-checkout: | + test + + - name: Generate testing id + run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.APM_E2E_TEST_ROLE_ARN }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + # local directory to store the kubernetes config + - name: Create kubeconfig directory + run: mkdir -p ${{ github.workspace }}/.kube + + - name: Set KUBECONFIG environment variable + run: echo KUBECONFIG="${{ github.workspace }}/.kube/config" >> $GITHUB_ENV + + - name: Install eksctl + run: | + mkdir ${{ github.workspace }}/eksctl + curl -sLO "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz" + tar -xzf eksctl_Linux_amd64.tar.gz -C ${{ github.workspace }}/eksctl && rm eksctl_Linux_amd64.tar.gz + echo "${{ github.workspace }}/eksctl" >> $GITHUB_PATH + + # Create the K8s service account to grant the sample app + # permissions to call AWS services from within the pods. + # https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + # + # This needs to be done before launching the sample app + # because the sample app manifest requires the service + # account to be created upfront. + - name: Enable OIDC for cluster + run: eksctl utils associate-iam-oidc-provider --cluster ${{ inputs.test-cluster-name }} --region ${{ env.AWS_DEFAULT_REGION }} --approve + + - name: Create role for AWS access from the sample app + id: create_service_account + run: | + eksctl create iamserviceaccount \ + --name service-account-${{ env.TESTING_ID }} \ + --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ + --cluster ${{ inputs.test-cluster-name }} \ + --role-name eks-s3-access-${{ env.TESTING_ID }} \ + --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \ + --region ${{ env.AWS_DEFAULT_REGION }} \ + --approve + + - name: Set up terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_wrapper: false + + - name: Deploy sample app via terraform + working-directory: test/terraform/eks + run: | + terraform init + terraform validate + terraform apply -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="kube_directory_path=${{ github.workspace }}/.kube" \ + -var="eks_cluster_name=${{ inputs.test-cluster-name }}" \ + -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \ + -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \ + -var="sample_app_image=${{ secrets.APM_E2E_SAMPLE_APP_FRONTEND_SERVICE_IMAGE }}" \ + -var="sample_remote_app_image=${{ secrets.APM_E2E_SAMPLE_APP_REMOTE_SERVICE_IMAGE }}" + + # Enable APM on the test cluster + - name: Pull and unzip enablement script from S3 + run: aws s3 cp ${{ secrets.APM_E2E_ONBOARDING_ZIP_S3_URI }} . && unzip -j onboarding.zip + + - name: Change OTEL_TRACES_SAMPLER to always_on + run: "sed -i 's#parentbased_traceidratio#always_on#g' instrumentation.yaml" + + - name: Set the CW Agent image in the manifest + run: yq -i '.spec.image = "${{ inputs.apm-cwagent-image-name }}"' agent-daemon-set.yaml + + - name: Enable APM + run: | + ./enable-apm.sh \ + ${{ inputs.test-cluster-name }} \ + ${{ env. AWS_DEFAULT_REGION }} \ + ${{ env.SAMPLE_APP_NAMESPACE }} + + # Application pods need to be restarted for the + # apm instrumentation to take effect + - name: Restart the app pods + run: kubectl delete pods --all -n ${{ env.SAMPLE_APP_NAMESPACE }} + + - name: Wait for sample app pods to come up + run: | + kubectl wait --for=condition=Ready pod --all -n ${{ env.SAMPLE_APP_NAMESPACE }} \ + + - name: Verify pod ADOT image + run: | + kubectl get pods -n ${{ env.SAMPLE_APP_NAMESPACE }} --output json | \ + jq '.items[0].status.initContainerStatuses[0].imageID' + + - name: Verify pod CWAgent image + run: | + kubectl get pods -n amazon-cloudwatch --output json | \ + jq '.items[0].status.containerStatuses[0].imageID' + + - name: Get the sample app endpoint + run: | + echo "APP_ENDPOINT=$(terraform output sample_app_endpoint)" >> $GITHUB_ENV + working-directory: test/terraform/eks + + - name: Wait for app endpoint to come online + run: | + attempt_counter=0 + max_attempts=12 + until $(curl --output /dev/null --silent --head --fail http://${{ env.APP_ENDPOINT }}); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$(($attempt_counter+1)) + sleep 10 + done + + # Validation for pulse telemetry data + - name: Call endpoint and validate generated EMF logs + id: log-validation + working-directory: test/validator + run: ./gradlew run --args='-c log-validation.yml + --testing-id ${{ env.TESTING_ID }} + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ secrets.APM_E2E_TEST_ACCOUNT }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --cluster ${{ inputs.test-cluster-name }} + --service-name sample-application-${{ env.TESTING_ID }} + --rollup' + + - name: Call endpoints and validate generated metrics + id: metric-validation + if: success() || steps.log-validation.outcome == 'failure' + working-directory: test/validator + run: ./gradlew run --args='-c metric-validation.yml + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ secrets.APM_E2E_TEST_ACCOUNT }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --cluster ${{ inputs.test-cluster-name }} + --service-name sample-application-${{ env.TESTING_ID }} + --remote-service-name sample-remote-application-${{ env.TESTING_ID }} + --rollup' + + - name: Call endpoints and validate generated traces + if: success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' + working-directory: test/validator + run: ./gradlew run --args='-c trace-validation.yml + --endpoint http://${{ env.APP_ENDPOINT }} + --region ${{ env.AWS_DEFAULT_REGION }} + --account-id ${{ secrets.APM_E2E_TEST_ACCOUNT }} + --metric-namespace ${{ env.METRIC_NAMESPACE }} + --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} + --cluster ${{ inputs.test-cluster-name }} + --service-name sample-application-${{ env.TESTING_ID }} + --remote-service-name sample-remote-application-${{ env.TESTING_ID }} + --rollup' + + # Clean up Procedures + + - name: Remove log group deletion command + if: always() + run: | + delete_log_group="aws logs delete-log-group --log-group-name '${{ env.APM_LOG_GROUP }}' --region \$REGION" + sed -i "s#$delete_log_group##g" clean-apm.sh + + - name: Clean APM + if: always() + run: | + ./clean-apm.sh \ + ${{ inputs.test-cluster-name }} \ + ${{ env. AWS_DEFAULT_REGION }} \ + ${{ env.SAMPLE_APP_NAMESPACE }} + + # This step also deletes lingering resources from previous test runs + - name: Delete all sample app resources + if: always() + continue-on-error: true + timeout-minutes: 10 + run: kubectl delete namespace ${{ env.SAMPLE_APP_NAMESPACE }} + + - name: Terraform destroy + if: always() + continue-on-error: true + run: | + cd test/terraform/eks + terraform destroy -auto-approve \ + -var="test_id=${{ env.TESTING_ID }}" \ + -var="kube_directory_path=${{ github.workspace }}/.kube" \ + -var="eks_cluster_name=${{ inputs.test-cluster-name }}" \ + -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \ + -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \ + -var="sample_app_image=${{ env.SAMPLE_APP_IMAGE }}" + + - name: Remove aws access service account + if: always() + continue-on-error: true + run: | + eksctl delete iamserviceaccount \ + --name service-account-${{ env.TESTING_ID }} \ + --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ + --cluster ${{ inputs.test-cluster-name }} \ + --region ${{ env.AWS_DEFAULT_REGION }} \ \ No newline at end of file From 3f1bc69cde4cc50c362c045dbf7df7719b315989 Mon Sep 17 00:00:00 2001 From: Hyunsoo Kim <884273+movence@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:13:25 -0400 Subject: [PATCH 06/38] OTLP port updates (#508) --- translator/translate/otel/receiver/otlp/apm_config.yaml | 4 ++-- translator/translate/otel/receiver/otlp/translator_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/translator/translate/otel/receiver/otlp/apm_config.yaml b/translator/translate/otel/receiver/otlp/apm_config.yaml index c14c705d9d..560470c3a2 100644 --- a/translator/translate/otel/receiver/otlp/apm_config.yaml +++ b/translator/translate/otel/receiver/otlp/apm_config.yaml @@ -1,5 +1,5 @@ protocols: grpc: - endpoint: 0.0.0.0:4317 + endpoint: 0.0.0.0:4315 http: - endpoint: 0.0.0.0:4318 \ No newline at end of file + endpoint: 0.0.0.0:4316 \ No newline at end of file diff --git a/translator/translate/otel/receiver/otlp/translator_test.go b/translator/translate/otel/receiver/otlp/translator_test.go index 42049ebe64..c7ba2385d8 100644 --- a/translator/translate/otel/receiver/otlp/translator_test.go +++ b/translator/translate/otel/receiver/otlp/translator_test.go @@ -92,10 +92,10 @@ func TestTranslateAPM(t *testing.T) { want: confmap.NewFromStringMap(map[string]interface{}{ "protocols": map[string]interface{}{ "grpc": map[string]interface{}{ - "endpoint": "0.0.0.0:4317", + "endpoint": "0.0.0.0:4315", }, "http": map[string]interface{}{ - "endpoint": "0.0.0.0:4318", + "endpoint": "0.0.0.0:4316", }, }, }), From 991c75e293456eb56d693253b2418dd74d5e1996 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 29 Sep 2023 16:55:19 -0400 Subject: [PATCH 07/38] Support APM for generic (non kubernetes) platforms (#519) Co-authored-by: Mengyi Zhou (bjrara) --- .../internal/resolver/attributesresolver.go | 36 +++++- .../awsapmprocessor/internal/resolver/eks.go | 78 ++++++++---- .../internal/resolver/eks_test.go | 4 +- .../internal/resolver/types.go | 18 +++ translator/translate/otel/common/apm.go | 13 ++ translator/translate/otel/common/apm_test.go | 16 +++ .../otel/exporter/awsemf/apm_config_eks.yaml | 34 ++++++ .../exporter/awsemf/apm_config_generic.yaml | 29 +++++ .../exporter/awsemf/awsemf_default_apm.yaml | 34 ------ .../otel/exporter/awsemf/translator.go | 19 +-- .../otel/exporter/awsemf/translator_test.go | 114 +++++++++--------- .../exporter/awsxray/awsxray_default_apm.yaml | 2 +- .../otel/exporter/awsxray/translator_test.go | 5 +- .../testdata/{config.yaml => config_eks.yaml} | 0 .../awsapm/testdata/config_generic.yaml | 1 + ...esConfig.yaml => validRulesConfigEKS.yaml} | 0 .../testdata/validRulesConfigGeneric.yaml | 28 +++++ .../otel/processor/awsapm/translator.go | 6 +- .../otel/processor/awsapm/translator_test.go | 49 ++++++-- 19 files changed, 342 insertions(+), 144 deletions(-) create mode 100644 processor/awsapmprocessor/internal/resolver/types.go create mode 100644 translator/translate/otel/common/apm.go create mode 100644 translator/translate/otel/common/apm_test.go create mode 100644 translator/translate/otel/exporter/awsemf/apm_config_eks.yaml create mode 100644 translator/translate/otel/exporter/awsemf/apm_config_generic.yaml delete mode 100644 translator/translate/otel/exporter/awsemf/awsemf_default_apm.yaml rename translator/translate/otel/processor/awsapm/testdata/{config.yaml => config_eks.yaml} (100%) create mode 100644 translator/translate/otel/processor/awsapm/testdata/config_generic.yaml rename translator/translate/otel/processor/awsapm/testdata/{validRulesConfig.yaml => validRulesConfigEKS.yaml} (100%) create mode 100644 translator/translate/otel/processor/awsapm/testdata/validRulesConfigGeneric.yaml diff --git a/processor/awsapmprocessor/internal/resolver/attributesresolver.go b/processor/awsapmprocessor/internal/resolver/attributesresolver.go index c7f243daa9..0b4525d38a 100644 --- a/processor/awsapmprocessor/internal/resolver/attributesresolver.go +++ b/processor/awsapmprocessor/internal/resolver/attributesresolver.go @@ -10,6 +10,10 @@ import ( "go.uber.org/zap" ) +var DefaultHostedInAttributes = map[string]string{ + AttributeHostedInEnvironment: HostedInAttributeEnvironment, +} + type subResolver interface { Process(attributes, resourceAttributes pcommon.Map) error Stop(ctx context.Context) error @@ -24,7 +28,9 @@ func NewAttributesResolver(resolverNames []string, logger *zap.Logger) *attribut subResolvers := []subResolver{} for _, resolverName := range resolverNames { if resolverName == "eks" { - subResolvers = append(subResolvers, getEksResolver(logger)) + subResolvers = append(subResolvers, getEksResolver(logger), newEKSHostedInAttributeResolver()) + } else { + subResolvers = append(subResolvers, newHostedInAttributeResolver(DefaultHostedInAttributes)) } } return &attributesResolver{ @@ -50,3 +56,31 @@ func (r *attributesResolver) Stop(ctx context.Context) error { } return nil } + +type hostedInAttributeResolver struct { + attributeMap map[string]string +} + +func newHostedInAttributeResolver(attributeMap map[string]string) *hostedInAttributeResolver { + return &hostedInAttributeResolver{ + attributeMap: attributeMap, + } +} +func (h *hostedInAttributeResolver) Process(attributes, resourceAttributes pcommon.Map) error { + for attrKey, mappingKey := range h.attributeMap { + if val, ok := resourceAttributes.Get(attrKey); ok { + attributes.PutStr(mappingKey, val.AsString()) + } + } + + if _, ok := resourceAttributes.Get(AttributeHostedInEnvironment); !ok { + hostedInEnv := "Generic" + attributes.PutStr(HostedInAttributeEnvironment, hostedInEnv) + } + + return nil +} + +func (h *hostedInAttributeResolver) Stop(ctx context.Context) error { + return nil +} diff --git a/processor/awsapmprocessor/internal/resolver/eks.go b/processor/awsapmprocessor/internal/resolver/eks.go index ee5ae21880..6f65f24f2e 100644 --- a/processor/awsapmprocessor/internal/resolver/eks.go +++ b/processor/awsapmprocessor/internal/resolver/eks.go @@ -17,6 +17,7 @@ import ( mapset "github.com/deckarep/golang-set/v2" "go.opentelemetry.io/collector/pdata/pcommon" + semconv "go.opentelemetry.io/collector/semconv/v1.17.0" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" @@ -42,6 +43,10 @@ const ( deletionDelay = 2 * time.Minute ) +var DefaultHostedInAttributeMap = map[string]string{ + semconv.AttributeK8SNamespaceName: HostedInAttributeK8SNamespace, +} + var ( // ReplicaSet name = Deployment name + "-" + 10 alphanumeric characters long string (see https://stackoverflow.com/questions/46204504/kubernetes-pod-naming-convention and https://stackoverflow.com/questions/71090356/how-does-random-string-in-kubernetes-pod-name-decided) // the alphanumeric characters in Kubernetes are restricted to exclude vowels to reduce the chances of "normal words" being formed. (see https://github.com/kubernetes/apimachinery/blob/master/pkg/util/rand/rand.go#L83) @@ -54,7 +59,6 @@ var ( ) type eksResolver struct { - clusterName string // used to save the cluster name as a cache logger *zap.Logger clientset kubernetes.Interface ipToPod *sync.Map @@ -621,13 +625,13 @@ func (e *eksResolver) debug() { } func (e *eksResolver) Process(attributes, resourceAttributes pcommon.Map) error { - if value, ok := attributes.Get("aws.remote.service"); ok { + if value, ok := attributes.Get(AttributeRemoteService); ok { valueStr := value.AsString() ipStr := "" if ip, _, ok := extractIPPort(valueStr); ok { if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(valueStr); err == nil { - attributes.PutStr("aws.remote.service", workload) - attributes.PutStr("K8s.RemoteNamespace", namespace) + attributes.PutStr(AttributeRemoteService, workload) + attributes.PutStr(AttributeRemoteNamespace, namespace) } else { ipStr = ip } @@ -637,32 +641,15 @@ func (e *eksResolver) Process(attributes, resourceAttributes pcommon.Map) error if ipStr != "" { if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(ipStr); err == nil { - attributes.PutStr("aws.remote.service", workload) - attributes.PutStr("K8s.RemoteNamespace", namespace) + attributes.PutStr(AttributeRemoteService, workload) + attributes.PutStr(AttributeRemoteNamespace, namespace) } else { e.logger.Debug("failed to Process ip", zap.String("ip", ipStr), zap.Error(err)) - attributes.PutStr("aws.remote.service", "UnknownRemoteService") + attributes.PutStr(AttributeRemoteService, "UnknownRemoteService") } } } - if namespace, ok := resourceAttributes.Get("k8s.namespace.name"); ok { - attributes.PutStr("K8s.Namespace", namespace.AsString()) - // assuming cluster name resource attribute like "ec2.tag.kubernetes.io/cluster/petclinic-test" always exist when "k8s.namespace.name" exist - if e.clusterName != "" { - attributes.PutStr("EKS.Cluster", e.clusterName) - } else { - // iterate resource attributes to find the cluster name - resourceAttributes.Range(func(key string, value pcommon.Value) bool { - if strings.HasPrefix(key, "ec2.tag.kubernetes.io/cluster/") && value.Type() == pcommon.ValueTypeStr && value.AsString() == "owned" { - e.clusterName = strings.TrimPrefix(key, "ec2.tag.kubernetes.io/cluster/") - attributes.PutStr("EKS.Cluster", e.clusterName) - return false - } - return true - }) - } - } return nil } @@ -707,3 +694,46 @@ func getHostNetworkPorts(pod *corev1.Pod) []string { } return ports } + +type eksHostedInAttributeResolver struct { + clusterName string + attributeMap map[string]string +} + +func newEKSHostedInAttributeResolver() *eksHostedInAttributeResolver { + return &eksHostedInAttributeResolver{ + attributeMap: map[string]string{ + semconv.AttributeK8SNamespaceName: HostedInAttributeK8SNamespace, + }, + } +} +func (h *eksHostedInAttributeResolver) Process(attributes, resourceAttributes pcommon.Map) error { + for attrKey, mappingKey := range h.attributeMap { + if val, ok := resourceAttributes.Get(attrKey); ok { + attributes.PutStr(mappingKey, val.AsString()) + } + } + + if h.clusterName != "" { + attributes.PutStr(HostedInAttributeClusterName, h.clusterName) + } else { + platform, _ := resourceAttributes.Get(semconv.AttributeCloudProvider) + if platform.AsString() == semconv.AttributeCloudProviderAWS { + // iterate resource attributes to find the cluster name + resourceAttributes.Range(func(key string, value pcommon.Value) bool { + if strings.HasPrefix(key, "ec2.tag.kubernetes.io/cluster/") && value.Type() == pcommon.ValueTypeStr && value.AsString() == "owned" { + h.clusterName = strings.TrimPrefix(key, "ec2.tag.kubernetes.io/cluster/") + attributes.PutStr(HostedInAttributeClusterName, h.clusterName) + return false + } + return true + }) + } + } + + return nil +} + +func (h *eksHostedInAttributeResolver) Stop(ctx context.Context) error { + return nil +} diff --git a/processor/awsapmprocessor/internal/resolver/eks_test.go b/processor/awsapmprocessor/internal/resolver/eks_test.go index 984672fc89..7da36d9c99 100644 --- a/processor/awsapmprocessor/internal/resolver/eks_test.go +++ b/processor/awsapmprocessor/internal/resolver/eks_test.go @@ -809,8 +809,8 @@ func TestEksResolver(t *testing.T) { resourceAttributes.PutStr("ec2.tag.kubernetes.io/cluster/test-cluster", "owned") err = resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, "K8s.Namespace", t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, "EKS.Cluster", t)) + assert.Equal(t, "test-namespace-3", getStrAttr(attributes, "HostedIn.K8s.Namespace", t)) + assert.Equal(t, "test-cluster", getStrAttr(attributes, "HostedIn.EKS.Cluster", t)) // Test case 6: Process with valid IP but GetWorkloadAndNamespaceByIP returns error attributes = pcommon.NewMap() diff --git a/processor/awsapmprocessor/internal/resolver/types.go b/processor/awsapmprocessor/internal/resolver/types.go new file mode 100644 index 0000000000..dbaa8fa7c1 --- /dev/null +++ b/processor/awsapmprocessor/internal/resolver/types.go @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resolver + +const ( + AttributeRemoteService = "aws.remote.service" + AttributeHostedInEnvironment = "aws.hostedin.environment" +) + +const ( + AttributeRemoteNamespace = "K8s.RemoteNamespace" +) +const ( + HostedInAttributeClusterName = "HostedIn.EKS.Cluster" + HostedInAttributeK8SNamespace = "HostedIn.K8s.Namespace" + HostedInAttributeEnvironment = "HostedIn.Environment" +) diff --git a/translator/translate/otel/common/apm.go b/translator/translate/otel/common/apm.go new file mode 100644 index 0000000000..5c6b4ab325 --- /dev/null +++ b/translator/translate/otel/common/apm.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package common + +import "os" + +const KubernetesEnvVar = "K8S_NAMESPACE" + +func IsAPMKubernetes() bool { + _, isSet := os.LookupEnv(KubernetesEnvVar) + return isSet +} diff --git a/translator/translate/otel/common/apm_test.go b/translator/translate/otel/common/apm_test.go new file mode 100644 index 0000000000..2e04f5da79 --- /dev/null +++ b/translator/translate/otel/common/apm_test.go @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsAPMKubernetes(t *testing.T) { + assert.False(t, IsAPMKubernetes()) + t.Setenv(KubernetesEnvVar, "TEST") + assert.True(t, IsAPMKubernetes()) +} diff --git a/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml b/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml new file mode 100644 index 0000000000..dba6bbc75c --- /dev/null +++ b/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml @@ -0,0 +1,34 @@ +log_group_name: "/aws/apm/eks" +namespace: "AWS/APM" +dimension_rollup_option: "NoDimensionRollup" +metric_declarations: + - dimensions: + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service] + label_matchers: + - label_names: + - aws.span.kind + regex: '^(SERVER|CONSUMER|LOCAL_ROOT)$' + metric_name_selectors: + - Latency + - Fault + - Error + - dimensions: + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, Operation, RemoteService, RemoteOperation] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, K8s.RemoteNamespace] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, RemoteTarget] + - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation] + label_matchers: + - label_names: + - aws.span.kind + regex: '^(CLIENT|PRODUCER)$' + metric_name_selectors: + - Latency + - Fault + - Error \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml b/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml new file mode 100644 index 0000000000..d1118e9f75 --- /dev/null +++ b/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml @@ -0,0 +1,29 @@ +log_group_name: "/aws/apm/generic" +namespace: "AWS/APM" +dimension_rollup_option: "NoDimensionRollup" +metric_declarations: + - dimensions: + - [HostedIn.Environment, Service, Operation] + - [HostedIn.Environment, Service] + label_matchers: + - label_names: + - aws.span.kind + regex: '^(SERVER|CONSUMER)$' + metric_name_selectors: + - Latency + - Fault + - Error + - dimensions: + - [HostedIn.Environment, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] + - [HostedIn.Environment, Service, Operation, RemoteService, RemoteOperation] + - [HostedIn.Environment, Service, RemoteService] + - [HostedIn.Environment, Service, RemoteService, RemoteOperation, RemoteTarget] + - [HostedIn.Environment, Service, RemoteService, RemoteOperation] + label_matchers: + - label_names: + - aws.span.kind + regex: '^(CLIENT|PRODUCER)$' + metric_name_selectors: + - Latency + - Fault + - Error \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/awsemf_default_apm.yaml b/translator/translate/otel/exporter/awsemf/awsemf_default_apm.yaml deleted file mode 100644 index d5256187c6..0000000000 --- a/translator/translate/otel/exporter/awsemf/awsemf_default_apm.yaml +++ /dev/null @@ -1,34 +0,0 @@ -log_group_name: "/aws/apm/eks" -namespace: "AWS/APM" -dimension_rollup_option: "NoDimensionRollup" -metric_declarations: - - dimensions: - - [EKS.Cluster, K8s.Namespace, Service, Operation] - - [EKS.Cluster, K8s.Namespace, Service] - label_matchers: - - label_names: - - aws.span.kind - regex: '^(SERVER|CONSUMER|LOCAL_ROOT)$' - metric_name_selectors: - - Latency - - Fault - - Error - - dimensions: - - [EKS.Cluster, K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [EKS.Cluster, K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [EKS.Cluster, K8s.Namespace, Service, Operation, RemoteService, RemoteOperation, RemoteTarget] - - [EKS.Cluster, K8s.Namespace, Service, Operation, RemoteService, RemoteOperation] - - [EKS.Cluster, K8s.Namespace, Service, RemoteService, K8s.RemoteNamespace] - - [EKS.Cluster, K8s.Namespace, Service, RemoteService] - - [EKS.Cluster, K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace, RemoteTarget] - - [EKS.Cluster, K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace] - - [EKS.Cluster, K8s.Namespace, Service, RemoteService, RemoteOperation, RemoteTarget] - - [EKS.Cluster, K8s.Namespace, Service, RemoteService, RemoteOperation] - label_matchers: - - label_names: - - aws.span.kind - regex: '^(CLIENT|PRODUCER)$' - metric_name_selectors: - - Latency - - Fault - - Error \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 962490e101..7db09cce45 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -28,8 +28,11 @@ var defaultKubernetesConfig string //go:embed awsemf_default_prometheus.yaml var defaultPrometheusConfig string -//go:embed awsemf_default_apm.yaml -var defaultAPMConfig string +//go:embed apm_config_eks.yaml +var apmConfigEks string + +//go:embed apm_config_generic.yaml +var apmConfigGeneric string var ( ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) @@ -61,6 +64,12 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*awsemfexporter.Config) + if common.IsAPMKubernetes() && t.name == common.APM { + return common.GetYamlFileToYamlConfig(cfg, apmConfigEks) + } else if t.name == common.APM { + return common.GetYamlFileToYamlConfig(cfg, apmConfigGeneric) + } + var defaultConfig string if isEcs(c) { defaultConfig = defaultEcsConfig @@ -68,8 +77,6 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { defaultConfig = defaultKubernetesConfig } else if isPrometheus(c) { defaultConfig = defaultPrometheusConfig - } else if isAPM(c) { - defaultConfig = defaultAPMConfig } else { return cfg, nil } @@ -109,10 +116,6 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { return cfg, nil } -func isAPM(conf *confmap.Conf) bool { - return conf.IsSet(common.APMMetrics) -} - func isEcs(conf *confmap.Conf) bool { return conf.IsSet(ecsBasePathKey) } diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index b93ac3272a..f58651a3f9 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -4,14 +4,19 @@ package awsemf import ( + "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" legacytranslator "github.com/aws/amazon-cloudwatch-agent/translator" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) var nilSlice []string @@ -583,65 +588,6 @@ func TestTranslator(t *testing.T) { "metric_descriptors": nilMetricDescriptorsSlice, }, }, - "WithAPMEnabled": { - input: map[string]interface{}{ - "logs": map[string]interface{}{ - "metrics_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, - }, - }}, - want: map[string]interface{}{ - "namespace": "AWS/APM", - "log_group_name": "/aws/apm/eks", - "log_stream_name": "", - "dimension_rollup_option": "NoDimensionRollup", - "disable_metric_extraction": false, - "enhanced_container_insights": false, - "parse_json_encoded_attr_values": nilSlice, - "output_destination": "cloudwatch", - "eks_fargate_container_insights_enabled": false, - "resource_to_telemetry_conversion": resourcetotelemetry.Settings{ - Enabled: false, - }, - "metric_declarations": []*awsemfexporter.MetricDeclaration{ - { - Dimensions: [][]string{ - {"EKS.Cluster", "K8s.Namespace", "Service", "Operation"}, - {"EKS.Cluster", "K8s.Namespace", "Service"}, - }, - LabelMatchers: []*awsemfexporter.LabelMatcher{ - { - LabelNames: []string{"aws.span.kind"}, - Regex: "^(SERVER|CONSUMER|LOCAL_ROOT)$", - }, - }, - MetricNameSelectors: []string{"Latency", "Fault", "Error"}, - }, - { - Dimensions: [][]string{ - {"EKS.Cluster", "K8s.Namespace", "Service", "Operation", "RemoteService", "RemoteOperation", "K8s.RemoteNamespace", "RemoteTarget"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "Operation", "RemoteService", "RemoteOperation", "K8s.RemoteNamespace"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "Operation", "RemoteService", "RemoteOperation", "RemoteTarget"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "Operation", "RemoteService", "RemoteOperation"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "K8s.RemoteNamespace"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "RemoteOperation", "K8s.RemoteNamespace", "RemoteTarget"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "RemoteOperation", "K8s.RemoteNamespace"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "RemoteOperation", "RemoteTarget"}, - {"EKS.Cluster", "K8s.Namespace", "Service", "RemoteService", "RemoteOperation"}, - }, - LabelMatchers: []*awsemfexporter.LabelMatcher{ - { - LabelNames: []string{"aws.span.kind"}, - Regex: "^(CLIENT|PRODUCER)$", - }, - }, - MetricNameSelectors: []string{"Latency", "Fault", "Error"}, - }, - }, - "metric_descriptors": nilMetricDescriptorsSlice, - }, - }, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { @@ -669,3 +615,53 @@ func TestTranslator(t *testing.T) { }) } } + +func TestTranslateAPM(t *testing.T) { + tt := NewTranslatorWithName(common.APM) + testCases := map[string]struct { + input map[string]interface{} + want *confmap.Conf + wantErr error + isKubernetes bool + }{ + "WithAPMEnabledEKS": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }}, + want: testutil.GetConf(t, filepath.Join("apm_config_eks.yaml")), + isKubernetes: true, + }, + "WithAPMEnabledGeneric": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }}, + want: testutil.GetConf(t, filepath.Join("apm_config_generic.yaml")), + isKubernetes: false, + }, + } + factory := awsemfexporter.NewFactory() + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + if testCase.isKubernetes { + t.Setenv(common.KubernetesEnvVar, "TEST") + } + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + assert.Equal(t, testCase.wantErr, err) + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*awsemfexporter.Config) + require.True(t, ok) + wantCfg := factory.CreateDefaultConfig() + require.NoError(t, component.UnmarshalConfig(testCase.want, wantCfg)) + assert.Equal(t, wantCfg, gotCfg) + } + }) + } +} diff --git a/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml b/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml index 7480bd4e2e..8cb4f29625 100644 --- a/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml +++ b/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml @@ -1,2 +1,2 @@ # Add new attributes as annotations for diagnostics. -indexed_attributes: [aws.local.service, aws.local.operation, aws.remote.service, aws.remote.operation, EKS.Cluster, K8s.Namespace, K8s.RemoteNamespace, aws.remote.target] \ No newline at end of file +indexed_attributes: [aws.local.service, aws.local.operation, aws.remote.service, aws.remote.operation, HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, K8s.RemoteNamespace, aws.remote.target, HostedIn.Environment] \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index 1ff05def06..35973acff0 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -64,10 +64,11 @@ func TestTranslator(t *testing.T) { "aws.local.operation", "aws.remote.service", "aws.remote.operation", - "EKS.Cluster", - "K8s.Namespace", + "HostedIn.EKS.Cluster", + "HostedIn.K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", + "HostedIn.Environment", }, "region": "us-east-1", "role_arn": "global_arn", diff --git a/translator/translate/otel/processor/awsapm/testdata/config.yaml b/translator/translate/otel/processor/awsapm/testdata/config_eks.yaml similarity index 100% rename from translator/translate/otel/processor/awsapm/testdata/config.yaml rename to translator/translate/otel/processor/awsapm/testdata/config_eks.yaml diff --git a/translator/translate/otel/processor/awsapm/testdata/config_generic.yaml b/translator/translate/otel/processor/awsapm/testdata/config_generic.yaml new file mode 100644 index 0000000000..7524660208 --- /dev/null +++ b/translator/translate/otel/processor/awsapm/testdata/config_generic.yaml @@ -0,0 +1 @@ +resolvers: ["generic"] \ No newline at end of file diff --git a/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.yaml b/translator/translate/otel/processor/awsapm/testdata/validRulesConfigEKS.yaml similarity index 100% rename from translator/translate/otel/processor/awsapm/testdata/validRulesConfig.yaml rename to translator/translate/otel/processor/awsapm/testdata/validRulesConfigEKS.yaml diff --git a/translator/translate/otel/processor/awsapm/testdata/validRulesConfigGeneric.yaml b/translator/translate/otel/processor/awsapm/testdata/validRulesConfigGeneric.yaml new file mode 100644 index 0000000000..d3453e91ee --- /dev/null +++ b/translator/translate/otel/processor/awsapm/testdata/validRulesConfigGeneric.yaml @@ -0,0 +1,28 @@ +resolvers: ["generic"] +rules: + - selectors: + - dimension: Operation + match: "POST *" + - dimension: RemoteService + match: "*" + action: keep + - selectors: + - dimension: Operation + match: "GET *" + - dimension: RemoteService + match: "*" + action: keep + rule_name: "keep02" + - selectors: + - dimension: Operation + match: "POST *" + action: drop + rule_name: "drop01" + - selectors: + - dimension: Operation + match: "*" + replacements: + - target_dimension: RemoteOperation + value: "This is a test string" + action: replace + rule_name: "replace01" \ No newline at end of file diff --git a/translator/translate/otel/processor/awsapm/translator.go b/translator/translate/otel/processor/awsapm/translator.go index 3d5bed51c9..6126778cba 100644 --- a/translator/translate/otel/processor/awsapm/translator.go +++ b/translator/translate/otel/processor/awsapm/translator.go @@ -57,7 +57,11 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { configKey := common.APMConfigKeys[t.dataType] cfg := t.factory.CreateDefaultConfig().(*awsapmprocessor.Config) - cfg.Resolvers = []string{"eks"} + if common.IsAPMKubernetes() { + cfg.Resolvers = []string{"eks"} + } else { + cfg.Resolvers = []string{"generic"} + } return t.translateCustomRules(conf, configKey, cfg) } diff --git a/translator/translate/otel/processor/awsapm/translator_test.go b/translator/translate/otel/processor/awsapm/translator_test.go index 7f7303c8a8..82cf6b2985 100644 --- a/translator/translate/otel/processor/awsapm/translator_test.go +++ b/translator/translate/otel/processor/awsapm/translator_test.go @@ -19,12 +19,16 @@ import ( ) var ( - //go:embed testdata/config.yaml - validAPMYaml string + //go:embed testdata/config_eks.yaml + validAPMYamlEKS string + //go:embed testdata/config_generic.yaml + validAPMYamlGeneric string //go:embed testdata/validRulesConfig.json validAPMRulesConfig string - //go:embed testdata/validRulesConfig.yaml - validAPMRulesYaml string + //go:embed testdata/validRulesConfigEKS.yaml + validAPMRulesYamlEKS string + //go:embed testdata/validRulesConfigGeneric.yaml + validAPMRulesYamlGeneric string //go:embed testdata/invalidRulesConfig.json invalidAPMRulesConfig string ) @@ -36,24 +40,42 @@ func TestTranslate(t *testing.T) { tt := NewTranslator(WithDataType(component.DataTypeMetrics)) testCases := map[string]struct { - input map[string]interface{} - want string - wantErr error + input map[string]interface{} + want string + wantErr error + isKubernetes bool }{ //The config for the awsapm processor is https://code.amazon.com/packages/AWSTracingSamplePetClinic/blobs/97ce3c409986ac8ae014de1e3fe71fdb98080f22/--/eks/apm/auto-instrumentation-new.yaml#L20 //The awsapm processor config does not have a platform field, instead it gets added to resolvers when marshalled - "WithAPMEnabled": { + "WithAPMEnabledEKS": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ "apm": map[string]interface{}{}, }, }}, - want: validAPMYaml, + want: validAPMYamlEKS, + isKubernetes: true, }, - "WithAPMCustomRulesEnabled": { - input: validJsonMap, - want: validAPMRulesYaml, + "WithAPMCustomRulesEnabledEKS": { + input: validJsonMap, + want: validAPMRulesYamlEKS, + isKubernetes: true, + }, + "WithAPMEnabledGeneric": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "apm": map[string]interface{}{}, + }, + }}, + want: validAPMYamlGeneric, + isKubernetes: false, + }, + "WithAPMCustomRulesEnabledGeneric": { + input: validJsonMap, + want: validAPMRulesYamlGeneric, + isKubernetes: false, }, "WithInvalidAPMCustomRulesEnabled": { input: invalidJsonMap, @@ -63,6 +85,9 @@ func TestTranslate(t *testing.T) { factory := awsapmprocessor.NewFactory() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { + if testCase.isKubernetes { + t.Setenv(common.KubernetesEnvVar, "TEST") + } conf := confmap.NewFromStringMap(testCase.input) got, err := tt.Translate(conf) assert.Equal(t, testCase.wantErr, err) From 728dbd937df4caae43022a387fc03cffe707c53a Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 10 Oct 2023 14:29:20 -0700 Subject: [PATCH 08/38] Add Remote Service an as Extra Metric Aggregation (#546) --- translator/translate/otel/exporter/awsemf/apm_config_eks.yaml | 1 + .../translate/otel/exporter/awsemf/apm_config_generic.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml b/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml index dba6bbc75c..aa6c3906c0 100644 --- a/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml +++ b/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml @@ -24,6 +24,7 @@ metric_declarations: - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, K8s.RemoteNamespace] - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation, RemoteTarget] - [HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, Service, RemoteService, RemoteOperation] + - [RemoteService] label_matchers: - label_names: - aws.span.kind diff --git a/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml b/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml index d1118e9f75..3488c6f65d 100644 --- a/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml +++ b/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml @@ -19,6 +19,7 @@ metric_declarations: - [HostedIn.Environment, Service, RemoteService] - [HostedIn.Environment, Service, RemoteService, RemoteOperation, RemoteTarget] - [HostedIn.Environment, Service, RemoteService, RemoteOperation] + - [RemoteService] label_matchers: - label_names: - aws.span.kind From 28fbd9d373e2040b33f05b7e8385878561f97a6a Mon Sep 17 00:00:00 2001 From: Thomas Pierce Date: Thu, 12 Oct 2023 11:39:30 -0700 Subject: [PATCH 09/38] Move CONSUMER spans to Dependency metrics (#555) --- translator/translate/otel/exporter/awsemf/apm_config_eks.yaml | 4 ++-- .../translate/otel/exporter/awsemf/apm_config_generic.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml b/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml index aa6c3906c0..8c87e3bd55 100644 --- a/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml +++ b/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml @@ -8,7 +8,7 @@ metric_declarations: label_matchers: - label_names: - aws.span.kind - regex: '^(SERVER|CONSUMER|LOCAL_ROOT)$' + regex: '^(SERVER|LOCAL_ROOT)$' metric_name_selectors: - Latency - Fault @@ -28,7 +28,7 @@ metric_declarations: label_matchers: - label_names: - aws.span.kind - regex: '^(CLIENT|PRODUCER)$' + regex: '^(CLIENT|PRODUCER|CONSUMER)$' metric_name_selectors: - Latency - Fault diff --git a/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml b/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml index 3488c6f65d..c95e805a09 100644 --- a/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml +++ b/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml @@ -8,7 +8,7 @@ metric_declarations: label_matchers: - label_names: - aws.span.kind - regex: '^(SERVER|CONSUMER)$' + regex: '^(SERVER|LOCAL_ROOT)$' metric_name_selectors: - Latency - Fault @@ -23,7 +23,7 @@ metric_declarations: label_matchers: - label_names: - aws.span.kind - regex: '^(CLIENT|PRODUCER)$' + regex: '^(CLIENT|PRODUCER|CONSUMER)$' metric_name_selectors: - Latency - Fault From 529e8f01253be90e621cca500879e2d6f29a5e18 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Thu, 19 Oct 2023 18:36:15 -0400 Subject: [PATCH 10/38] Fix generic unit test --- .../sampleConfig/base_apm_config.yaml | 71 +++++-------------- translator/tocwconfig/tocwconfig_test.go | 4 +- 2 files changed, 19 insertions(+), 56 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/base_apm_config.yaml b/translator/tocwconfig/sampleConfig/base_apm_config.yaml index 6c620f5eb7..1b4532a21e 100644 --- a/translator/tocwconfig/sampleConfig/base_apm_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_apm_config.yaml @@ -8,96 +8,57 @@ exporters: eks_fargate_container_insights_enabled: false endpoint: "" enhanced_container_insights: false - imds_retries: 1 + imds_retries: 0 local_mode: false - log_group_name: /aws/apm/eks + log_group_name: /aws/apm/generic log_retention: 0 log_stream_name: "" max_retries: 2 metric_declarations: - dimensions: - - - EKS.Cluster - - K8s.Namespace + - - HostedIn.Environment - Operation - Service - - - EKS.Cluster - - K8s.Namespace + - - HostedIn.Environment - Service label_matchers: - label_names: - aws.span.kind - regex: ^(SERVER|CONSUMER|LOCAL_ROOT)$ + regex: ^(SERVER|LOCAL_ROOT)$ separator: ; metric_name_selectors: - Latency - Fault - Error - dimensions: - - - EKS.Cluster - - K8s.Namespace - - K8s.RemoteNamespace + - - HostedIn.Environment - Operation - RemoteOperation - RemoteService - RemoteTarget - Service - - - EKS.Cluster - - K8s.Namespace - - K8s.RemoteNamespace - - Operation - - RemoteOperation - - RemoteService - - Service - - - EKS.Cluster - - K8s.Namespace + - - HostedIn.Environment - Operation - RemoteOperation - RemoteService - - RemoteTarget - - Service - - - EKS.Cluster - - K8s.Namespace - - Operation - - RemoteOperation - - RemoteService - - Service - - - EKS.Cluster - - K8s.Namespace - - K8s.RemoteNamespace - - RemoteService - - Service - - - EKS.Cluster - - K8s.Namespace - - RemoteService - - Service - - - EKS.Cluster - - K8s.Namespace - - K8s.RemoteNamespace - - RemoteOperation - - RemoteService - - RemoteTarget - Service - - - EKS.Cluster - - K8s.Namespace - - K8s.RemoteNamespace - - RemoteOperation + - - HostedIn.Environment - RemoteService - Service - - - EKS.Cluster - - K8s.Namespace + - - HostedIn.Environment - RemoteOperation - RemoteService - RemoteTarget - Service - - - EKS.Cluster - - K8s.Namespace + - - HostedIn.Environment - RemoteOperation - RemoteService - Service + - - RemoteService label_matchers: - label_names: - aws.span.kind - regex: ^(CLIENT|PRODUCER)$ + regex: ^(CLIENT|PRODUCER|CONSUMER)$ separator: ; metric_name_selectors: - Latency @@ -111,7 +72,7 @@ exporters: parse_json_encoded_attr_values: [] profile: "" proxy_address: "" - region: us-east-1 + region: "" request_timeout_seconds: 30 resource_arn: "" resource_to_telemetry_conversion: @@ -160,7 +121,7 @@ extensions: processors: awsapm: resolvers: - - eks + - generic rules: [] resourcedetection: aks: @@ -427,7 +388,7 @@ receivers: protocols: grpc: auth: null - endpoint: 0.0.0.0:4317 + endpoint: 0.0.0.0:4315 include_metadata: false keepalive: null max_concurrent_streams: 0 @@ -439,7 +400,7 @@ receivers: http: auth: null cors: null - endpoint: 0.0.0.0:4318 + endpoint: 0.0.0.0:4316 include_metadata: false logs_url_path: /v1/logs max_request_body_size: 0 diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index a34ab0d290..3211a74524 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "io/fs" "os" "path/filepath" @@ -56,7 +57,7 @@ func TestBaseContainerInsightsConfig(t *testing.T) { checkTranslation(t, "base_container_insights_config", "linux", expectedEnvVars, "") } -func TestBaseAPMConfig(t *testing.T) { +func TestGenericAPMConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(true) t.Setenv(config.HOST_NAME, "host_name_from_env") @@ -71,6 +72,7 @@ func TestAPMAndKubernetesConfig(t *testing.T) { context.CurrentContext().SetRunInContainer(true) t.Setenv(config.HOST_NAME, "host_name_from_env") t.Setenv(config.HOST_IP, "127.0.0.1") + t.Setenv(common.KubernetesEnvVar, "use_apm_eks_config") expectedEnvVars := map[string]string{} checkTranslation(t, "apm_and_kubernetes_config", "linux", expectedEnvVars, "") checkTranslation(t, "apm_and_kubernetes_config", "windows", expectedEnvVars, "") From fc7f42fb2bde4e6bbafd3df333185095536b11e9 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Thu, 19 Oct 2023 18:48:21 -0400 Subject: [PATCH 11/38] Fix apm and kubernetes config --- .../apm_and_kubernetes_config.yaml | 1278 +++++++++-------- 1 file changed, 640 insertions(+), 638 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml index 11b1190477..5c6253ba6e 100644 --- a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml @@ -1,646 +1,648 @@ connectors: {} exporters: - awsemf/apm: - certificate_file_path: "" - detailed_metrics: false - dimension_rollup_option: NoDimensionRollup - disable_metric_extraction: true - eks_fargate_container_insights_enabled: false - endpoint: "" - enhanced_container_insights: false - imds_retries: 1 - local_mode: false - log_group_name: /aws/containerinsights/{ClusterName}/performance - log_retention: 0 - log_stream_name: '{NodeName}' - max_retries: 2 - metric_declarations: - - dimensions: - - - ClusterName - - Namespace - - PodName - - - ClusterName - - - ClusterName - - Namespace - - Service - - - ClusterName - - Namespace - label_matchers: [] - metric_name_selectors: - - pod_cpu_utilization - - pod_memory_utilization - - pod_network_rx_bytes - - pod_network_tx_bytes - - pod_cpu_utilization_over_pod_limit - - pod_memory_utilization_over_pod_limit - - dimensions: - - - ClusterName - - Namespace - - PodName - label_matchers: [] - metric_name_selectors: - - pod_number_of_container_restarts - - dimensions: - - - ClusterName - - Namespace - - PodName - - - ClusterName - label_matchers: [] - metric_name_selectors: - - pod_cpu_reserved_capacity - - pod_memory_reserved_capacity - - dimensions: - - - ClusterName - - InstanceId - - NodeName - - - ClusterName - label_matchers: [] - metric_name_selectors: - - node_cpu_utilization - - node_memory_utilization - - node_network_total_bytes - - node_cpu_reserved_capacity - - node_memory_reserved_capacity - - node_number_of_running_pods - - node_number_of_running_containers - - dimensions: - - - ClusterName - label_matchers: [] - metric_name_selectors: - - node_cpu_usage_total - - node_cpu_limit - - node_memory_working_set - - node_memory_limit - - dimensions: - - - ClusterName - - InstanceId - - NodeName - - - ClusterName - label_matchers: [] - metric_name_selectors: - - node_filesystem_utilization - - dimensions: - - - ClusterName - - Namespace - - Service - - - ClusterName - label_matchers: [] - metric_name_selectors: - - service_number_of_running_pods - - dimensions: - - - ClusterName - - Namespace - - - ClusterName - label_matchers: [] - metric_name_selectors: - - namespace_number_of_running_pods - - dimensions: - - - ClusterName - label_matchers: [] - metric_name_selectors: - - cluster_node_count - - cluster_failed_node_count - metric_descriptors: [] - namespace: ContainerInsights - no_verify_ssl: false - num_workers: 8 - output_destination: cloudwatch - parse_json_encoded_attr_values: - - Sources - - kubernetes - profile: "" - proxy_address: "" - region: us-east-1 - request_timeout_seconds: 30 - resource_arn: "" - resource_to_telemetry_conversion: - enabled: true - retain_initial_value_of_delta_metric: false - role_arn: "" - shared_credentials_file: [] - version: "0" - awsemf/containerinsights: - certificate_file_path: "" - detailed_metrics: false - dimension_rollup_option: NoDimensionRollup - disable_metric_extraction: true - eks_fargate_container_insights_enabled: false - endpoint: "" - enhanced_container_insights: false - imds_retries: 1 - local_mode: false - log_group_name: /aws/containerinsights/{ClusterName}/performance - log_retention: 0 - log_stream_name: '{NodeName}' - max_retries: 2 - metric_declarations: - - dimensions: - - - ClusterName - - Namespace - - PodName - - - ClusterName - - - ClusterName - - Namespace - - Service - - - ClusterName - - Namespace - label_matchers: [] - metric_name_selectors: - - pod_cpu_utilization - - pod_memory_utilization - - pod_network_rx_bytes - - pod_network_tx_bytes - - pod_cpu_utilization_over_pod_limit - - pod_memory_utilization_over_pod_limit - - dimensions: - - - ClusterName - - Namespace - - PodName - label_matchers: [] - metric_name_selectors: - - pod_number_of_container_restarts - - dimensions: - - - ClusterName - - Namespace - - PodName - - - ClusterName - label_matchers: [] - metric_name_selectors: - - pod_cpu_reserved_capacity - - pod_memory_reserved_capacity - - dimensions: - - - ClusterName - - InstanceId - - NodeName - - - ClusterName - label_matchers: [] - metric_name_selectors: - - node_cpu_utilization - - node_memory_utilization - - node_network_total_bytes - - node_cpu_reserved_capacity - - node_memory_reserved_capacity - - node_number_of_running_pods - - node_number_of_running_containers - - dimensions: - - - ClusterName - label_matchers: [] - metric_name_selectors: - - node_cpu_usage_total - - node_cpu_limit - - node_memory_working_set - - node_memory_limit - - dimensions: - - - ClusterName - - InstanceId - - NodeName - - - ClusterName - label_matchers: [] - metric_name_selectors: - - node_filesystem_utilization - - dimensions: - - - ClusterName - - Namespace - - Service - - - ClusterName - label_matchers: [] - metric_name_selectors: - - service_number_of_running_pods - - dimensions: - - - ClusterName - - Namespace - - - ClusterName - label_matchers: [] - metric_name_selectors: - - namespace_number_of_running_pods - - dimensions: - - - ClusterName - label_matchers: [] - metric_name_selectors: - - cluster_node_count - - cluster_failed_node_count - metric_descriptors: [] - namespace: ContainerInsights - no_verify_ssl: false - num_workers: 8 - output_destination: cloudwatch - parse_json_encoded_attr_values: - - Sources - - kubernetes - profile: "" - proxy_address: "" - region: us-east-1 - request_timeout_seconds: 30 - resource_arn: "" - resource_to_telemetry_conversion: - enabled: true - retain_initial_value_of_delta_metric: false - role_arn: "" - shared_credentials_file: [] - version: "0" - awsxray/apm: - aws_log_groups: [] - certificate_file_path: "" - endpoint: "" - imds_retries: 1 - index_all_attributes: false - indexed_attributes: - - aws.local.service - - aws.local.operation - - aws.remote.service - - aws.remote.operation - - EKS.Cluster - - K8s.Namespace + awsemf/apm: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 0 + local_mode: false + log_group_name: /aws/apm/eks + log_retention: 0 + log_stream_name: "" + max_retries: 2 + metric_declarations: + - dimensions: + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - Operation + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - Service + label_matchers: + - label_names: + - aws.span.kind + regex: ^(SERVER|LOCAL_ROOT)$ + separator: ; + metric_name_selectors: + - Latency + - Fault + - Error + - dimensions: + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace - K8s.RemoteNamespace - - aws.remote.target - local_mode: false - max_retries: 2 - no_verify_ssl: false - num_workers: 8 - profile: "" - proxy_address: "" - region: us-east-1 - request_timeout_seconds: 30 - resource_arn: "" - role_arn: "" - shared_credentials_file: [] - telemetry: - enabled: true - include_metadata: true + - Operation + - RemoteOperation + - RemoteService + - RemoteTarget + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - K8s.RemoteNamespace + - Operation + - RemoteOperation + - RemoteService + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - Operation + - RemoteOperation + - RemoteService + - RemoteTarget + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - Operation + - RemoteOperation + - RemoteService + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - K8s.RemoteNamespace + - RemoteService + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - RemoteService + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - K8s.RemoteNamespace + - RemoteOperation + - RemoteService + - RemoteTarget + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - K8s.RemoteNamespace + - RemoteOperation + - RemoteService + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - RemoteOperation + - RemoteService + - RemoteTarget + - Service + - - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace + - RemoteOperation + - RemoteService + - Service + - - RemoteService + label_matchers: + - label_names: + - aws.span.kind + regex: ^(CLIENT|PRODUCER|CONSUMER)$ + separator: ; + metric_name_selectors: + - Latency + - Fault + - Error + metric_descriptors: [] + namespace: AWS/APM + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: [] + profile: "" + proxy_address: "" + region: "" + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: false + retain_initial_value_of_delta_metric: false + role_arn: "" + shared_credentials_file: [] + version: "1" + awsemf/containerinsights: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: true + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - Namespace + - Service + - - ClusterName + - Namespace + label_matchers: [] + metric_name_selectors: + - pod_cpu_utilization + - pod_memory_utilization + - pod_network_rx_bytes + - pod_network_tx_bytes + - pod_cpu_utilization_over_pod_limit + - pod_memory_utilization_over_pod_limit + - dimensions: + - - ClusterName + - Namespace + - PodName + label_matchers: [] + metric_name_selectors: + - pod_number_of_container_restarts + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - pod_cpu_reserved_capacity + - pod_memory_reserved_capacity + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_cpu_utilization + - node_memory_utilization + - node_network_total_bytes + - node_cpu_reserved_capacity + - node_memory_reserved_capacity + - node_number_of_running_pods + - node_number_of_running_containers + - dimensions: + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_cpu_usage_total + - node_cpu_limit + - node_memory_working_set + - node_memory_limit + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + label_matchers: [] + metric_name_selectors: + - node_filesystem_utilization + - dimensions: + - - ClusterName + - Namespace + - Service + - - ClusterName + label_matchers: [] + metric_name_selectors: + - service_number_of_running_pods + - dimensions: + - - ClusterName + - Namespace + - - ClusterName + label_matchers: [] + metric_name_selectors: + - namespace_number_of_running_pods + - dimensions: + - - ClusterName + label_matchers: [] + metric_name_selectors: + - cluster_node_count + - cluster_failed_node_count + metric_descriptors: [] + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + shared_credentials_file: [] + version: "0" + awsxray/apm: + aws_log_groups: [] + certificate_file_path: "" + endpoint: "" + imds_retries: 1 + index_all_attributes: false + indexed_attributes: + - aws.local.service + - aws.local.operation + - aws.remote.service + - aws.remote.operation + - EKS.Cluster + - K8s.Namespace + - K8s.RemoteNamespace + - aws.remote.target + local_mode: false + max_retries: 2 + no_verify_ssl: false + num_workers: 8 + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 30 + resource_arn: "" + role_arn: "" + shared_credentials_file: [] + telemetry: + enabled: true + include_metadata: true extensions: - awsproxy/apm: - aws_endpoint: "" - endpoint: 0.0.0.0:2000 - local_mode: false - proxy_address: "" - region: "" - role_arn: "" + awsproxy/apm: + aws_endpoint: "" + endpoint: 0.0.0.0:2000 + local_mode: false + proxy_address: "" + region: "" + role_arn: "" processors: - awsapm: - resolvers: - - eks - rules: [] - batch/containerinsights: - metadata_cardinality_limit: 1000 - metadata_keys: [] - send_batch_max_size: 0 - send_batch_size: 8192 - timeout: 5s - resourcedetection: - aks: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - attributes: [] + awsapm: + resolvers: + - eks + rules: [] + batch/containerinsights: + metadata_cardinality_limit: 1000 + metadata_keys: [] + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + resourcedetection: + aks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + attributes: [] + auth: null + azure: + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + compression: "" + consul: + address: "" + datacenter: "" + meta: {} + namespace: "" + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + token: '[REDACTED]' + token_file: "" + detectors: + - eks + - env + - ec2 + docker: + resource_attributes: + host.name: + enabled: true + os.type: + enabled: true + ec2: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.image.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + tags: + - ^kubernetes.io/cluster/.*$ + ecs: + resource_attributes: + aws.ecs.cluster.arn: + enabled: true + aws.ecs.launchtype: + enabled: true + aws.ecs.task.arn: + enabled: true + aws.ecs.task.family: + enabled: true + aws.ecs.task.revision: + enabled: true + aws.log.group.arns: + enabled: true + aws.log.group.names: + enabled: true + aws.log.stream.arns: + enabled: true + aws.log.stream.names: + enabled: true + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + eks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + elasticbeanstalk: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + deployment.environment: + enabled: true + service.instance.id: + enabled: true + service.version: + enabled: true + endpoint: "" + gcp: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.id: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + gcp.cloud_run.job.execution: + enabled: true + gcp.cloud_run.job.task_index: + enabled: true + gcp.gce.instance.hostname: + enabled: false + gcp.gce.instance.name: + enabled: false + host.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + k8s.cluster.name: + enabled: true + headers: {} + heroku: + resource_attributes: + cloud.provider: + enabled: true + heroku.app.id: + enabled: true + heroku.dyno.id: + enabled: true + heroku.release.commit: + enabled: true + heroku.release.creation_timestamp: + enabled: true + service.instance.id: + enabled: true + service.name: + enabled: true + service.version: + enabled: true + idle_conn_timeout: 1m30s + lambda: + resource_attributes: + aws.log.group.names: + enabled: true + aws.log.stream.names: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.instance: + enabled: true + faas.max_memory: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + max_conns_per_host: null + max_idle_conns: 100 + max_idle_conns_per_host: null + openshift: + address: "" + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + k8s.cluster.name: + enabled: true + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + token: "" + override: true + read_buffer_size: 0 + system: + hostname_sources: [] + resource_attributes: + host.arch: + enabled: false + host.id: + enabled: false + host.name: + enabled: true + os.description: + enabled: false + os.type: + enabled: true + timeout: 2s + write_buffer_size: 0 +receivers: + awscontainerinsightreceiver: + add_container_name_metric_label: false + add_full_pod_name_metric_label: false + add_service_as_attribute: true + certificate_file_path: "" + cluster_name: TestCluster + collection_interval: 30s + container_orchestrator: eks + enable_control_plane_metrics: false + endpoint: "" + imds_retries: 1 + leader_lock_name: cwagent-clusterleader + leader_lock_using_config_map_only: true + local_mode: false + max_retries: 0 + no_verify_ssl: false + num_workers: 0 + prefer_full_pod_name: false + profile: "" + proxy_address: "" + region: us-east-1 + request_timeout_seconds: 0 + resource_arn: "" + role_arn: "" + shared_credentials_file: [] + otlp/apm: + protocols: + grpc: auth: null - azure: - resource_attributes: - azure.resourcegroup.name: - enabled: true - azure.vm.name: - enabled: true - azure.vm.scaleset.name: - enabled: true - azure.vm.size: - enabled: true - cloud.account.id: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - compression: "" - consul: - address: "" - datacenter: "" - meta: {} - namespace: "" - resource_attributes: - azure.resourcegroup.name: - enabled: true - azure.vm.name: - enabled: true - azure.vm.scaleset.name: - enabled: true - azure.vm.size: - enabled: true - cloud.account.id: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - token: '[REDACTED]' - token_file: "" - detectors: - - eks - - env - - ec2 - docker: - resource_attributes: - host.name: - enabled: true - os.type: - enabled: true - ec2: - resource_attributes: - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.image.id: - enabled: true - host.name: - enabled: true - host.type: - enabled: true - tags: - - ^kubernetes.io/cluster/.*$ - ecs: - resource_attributes: - aws.ecs.cluster.arn: - enabled: true - aws.ecs.launchtype: - enabled: true - aws.ecs.task.arn: - enabled: true - aws.ecs.task.family: - enabled: true - aws.ecs.task.revision: - enabled: true - aws.log.group.arns: - enabled: true - aws.log.group.names: - enabled: true - aws.log.stream.arns: - enabled: true - aws.log.stream.names: - enabled: true - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - eks: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - elasticbeanstalk: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - deployment.environment: - enabled: true - service.instance.id: - enabled: true - service.version: - enabled: true - endpoint: "" - gcp: - resource_attributes: - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - faas.id: - enabled: true - faas.name: - enabled: true - faas.version: - enabled: true - gcp.cloud_run.job.execution: - enabled: true - gcp.cloud_run.job.task_index: - enabled: true - gcp.gce.instance.hostname: - enabled: false - gcp.gce.instance.name: - enabled: false - host.id: - enabled: true - host.name: - enabled: true - host.type: - enabled: true - k8s.cluster.name: - enabled: true - headers: {} - heroku: - resource_attributes: - cloud.provider: - enabled: true - heroku.app.id: - enabled: true - heroku.dyno.id: - enabled: true - heroku.release.commit: - enabled: true - heroku.release.creation_timestamp: - enabled: true - service.instance.id: - enabled: true - service.name: - enabled: true - service.version: - enabled: true - idle_conn_timeout: 1m30s - lambda: - resource_attributes: - aws.log.group.names: - enabled: true - aws.log.stream.names: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - faas.instance: - enabled: true - faas.max_memory: - enabled: true - faas.name: - enabled: true - faas.version: - enabled: true - max_conns_per_host: null - max_idle_conns: 100 - max_idle_conns_per_host: null - openshift: - address: "" - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - k8s.cluster.name: - enabled: true - tls: - ca_file: "" - ca_pem: '[REDACTED]' - cert_file: "" - cert_pem: '[REDACTED]' - insecure: false - insecure_skip_verify: false - key_file: "" - key_pem: '[REDACTED]' - max_version: "" - min_version: "" - reload_interval: 0s - server_name_override: "" - token: "" - override: true - read_buffer_size: 0 - system: - hostname_sources: [] - resource_attributes: - host.arch: - enabled: false - host.id: - enabled: false - host.name: - enabled: true - os.description: - enabled: false - os.type: - enabled: true - timeout: 2s + endpoint: 0.0.0.0:4315 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp write_buffer_size: 0 -receivers: - awscontainerinsightreceiver: - add_container_name_metric_label: false - add_full_pod_name_metric_label: false - add_service_as_attribute: true - certificate_file_path: "" - cluster_name: TestCluster - collection_interval: 30s - container_orchestrator: eks - enable_control_plane_metrics: false - endpoint: "" - imds_retries: 1 - leader_lock_name: cwagent-clusterleader - leader_lock_using_config_map_only: true - local_mode: false - max_retries: 0 - no_verify_ssl: false - num_workers: 0 - prefer_full_pod_name: false - profile: "" - proxy_address: "" - region: us-east-1 - request_timeout_seconds: 0 - resource_arn: "" - role_arn: "" - shared_credentials_file: [] - otlp/apm: - protocols: - grpc: - auth: null - endpoint: 0.0.0.0:4317 - include_metadata: false - keepalive: null - max_concurrent_streams: 0 - max_recv_msg_size_mib: 0 - read_buffer_size: 524288 - tls: null - transport: tcp - write_buffer_size: 0 - http: - auth: null - cors: null - endpoint: 0.0.0.0:4318 - include_metadata: false - logs_url_path: /v1/logs - max_request_body_size: 0 - metrics_url_path: /v1/metrics - response_headers: {} - tls: null - traces_url_path: /v1/traces + http: + auth: null + cors: null + endpoint: 0.0.0.0:4316 + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + response_headers: {} + tls: null + traces_url_path: /v1/traces service: - extensions: - - awsproxy/apm - pipelines: - metrics/apm: - exporters: - - awsemf/apm - processors: - - resourcedetection - - awsapm - receivers: - - otlp/apm - metrics/containerinsights: - exporters: - - awsemf/containerinsights - processors: - - batch/containerinsights - receivers: - - awscontainerinsightreceiver - traces/apm: - exporters: - - awsxray/apm - processors: - - resourcedetection - - awsapm - receivers: - - otlp/apm - telemetry: - logs: - development: false - disable_caller: false - disable_stacktrace: false - encoding: console - error_output_paths: [] - initial_fields: {} - level: info - output_paths: [] - sampling: - initial: 2 - thereafter: 500 - metrics: - address: "" - level: None - readers: [] - resource: {} - traces: - processors: [] - propagators: [] \ No newline at end of file + extensions: + - awsproxy/apm + pipelines: + metrics/apm: + exporters: + - awsemf/apm + processors: + - resourcedetection + - awsapm + receivers: + - otlp/apm + metrics/containerinsights: + exporters: + - awsemf/containerinsights + processors: + - batch/containerinsights + receivers: + - awscontainerinsightreceiver + traces/apm: + exporters: + - awsxray/apm + processors: + - resourcedetection + - awsapm + receivers: + - otlp/apm + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: [] + initial_fields: {} + level: info + output_paths: [] + sampling: + initial: 2 + thereafter: 500 + metrics: + address: "" + level: None + readers: [] + resource: {} + traces: + processors: [] + propagators: [] \ No newline at end of file From 65e278f6b0009fee9b28376d61c4892b876d3133 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Thu, 19 Oct 2023 18:48:53 -0400 Subject: [PATCH 12/38] make fmt --- translator/tocwconfig/tocwconfig_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 3211a74524..d989493e81 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "io/fs" "os" "path/filepath" @@ -17,6 +16,8 @@ import ( "strings" "testing" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "github.com/BurntSushi/toml" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" From 24a0e200f6e60b2fe9f0c56cd07d1bbcc9cc7735 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Thu, 19 Oct 2023 20:06:00 -0400 Subject: [PATCH 13/38] Fix imports --- translator/tocwconfig/tocwconfig_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index d989493e81..523c4cffce 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -16,8 +16,6 @@ import ( "strings" "testing" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" - "github.com/BurntSushi/toml" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -38,6 +36,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/tocwconfig/totomlconfig/tomlConfigTemplate" "github.com/aws/amazon-cloudwatch-agent/translator/tocwconfig/toyamlconfig" "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/util" ) From 53b53ed122c280f2f54f62e3a57510c1f2a55ea8 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 20 Oct 2023 10:58:09 -0400 Subject: [PATCH 14/38] Fix xray translator unit test --- .../otel/exporter/awsxray/awsxray_default_apm.yaml | 2 -- translator/translate/otel/exporter/awsxray/translator.go | 6 ++---- 2 files changed, 2 insertions(+), 6 deletions(-) delete mode 100644 translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml diff --git a/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml b/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml deleted file mode 100644 index 8cb4f29625..0000000000 --- a/translator/translate/otel/exporter/awsxray/awsxray_default_apm.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# Add new attributes as annotations for diagnostics. -indexed_attributes: [aws.local.service, aws.local.operation, aws.remote.service, aws.remote.operation, HostedIn.EKS.Cluster, HostedIn.K8s.Namespace, K8s.RemoteNamespace, aws.remote.target, HostedIn.Environment] \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsxray/translator.go b/translator/translate/otel/exporter/awsxray/translator.go index 06b1869e4c..9b11902630 100644 --- a/translator/translate/otel/exporter/awsxray/translator.go +++ b/translator/translate/otel/exporter/awsxray/translator.go @@ -22,9 +22,6 @@ const ( resourceARNKey = "resource_arn" ) -//go:embed awsxray_default_apm.yaml -var defaultAPMConfig string - type translator struct { name string factory exporter.Factory @@ -56,7 +53,8 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { if isAPM(conf) { cfg.IndexedAttributes = []string{ "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", - "EKS.Cluster", "K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", + "HostedIn.EKS.Cluster", "HostedIn.K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", + "HostedIn.Environment", } } From 9e1a234d473a6afd4011af401b4dc3e51ee08c66 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 20 Oct 2023 11:37:03 -0400 Subject: [PATCH 15/38] Fix xray translator unit test in tocwconfig --- .../tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml | 5 +++-- translator/tocwconfig/sampleConfig/base_apm_config.yaml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml index 5c6253ba6e..648a4abaeb 100644 --- a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml @@ -251,10 +251,11 @@ exporters: - aws.local.operation - aws.remote.service - aws.remote.operation - - EKS.Cluster - - K8s.Namespace + - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace - K8s.RemoteNamespace - aws.remote.target + - HostedIn.Environment local_mode: false max_retries: 2 no_verify_ssl: false diff --git a/translator/tocwconfig/sampleConfig/base_apm_config.yaml b/translator/tocwconfig/sampleConfig/base_apm_config.yaml index 1b4532a21e..7a11090e1c 100644 --- a/translator/tocwconfig/sampleConfig/base_apm_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_apm_config.yaml @@ -92,10 +92,11 @@ exporters: - aws.local.operation - aws.remote.service - aws.remote.operation - - EKS.Cluster - - K8s.Namespace + - HostedIn.EKS.Cluster + - HostedIn.K8s.Namespace - K8s.RemoteNamespace - aws.remote.target + - HostedIn.Environment local_mode: false max_retries: 2 no_verify_ssl: false From 8b227ed3719236915e2e8f7450545393dc6b5d3d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:59:47 -0400 Subject: [PATCH 16/38] Update OTel fork components to 8bdf732d320ad6718d88a6a42673081eb051d8a3 (#581) Co-authored-by: Github Action --- go.mod | 30 ++++++++++++++++-------------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 8a8785973b..88db517a48 100644 --- a/go.mod +++ b/go.mod @@ -6,34 +6,34 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2022 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a -replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230928170322-0df38c533713 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713 @@ -165,6 +165,7 @@ require ( go.opentelemetry.io/collector/config/configtelemetry v0.84.1-0.20230908201109-ab3d6c5b6470 go.opentelemetry.io/collector/extension v0.84.1-0.20230908201109-ab3d6c5b6470 go.opentelemetry.io/collector/processor v0.84.1-0.20230908201109-ab3d6c5b6470 + go.opentelemetry.io/collector/semconv v0.84.1-0.20230908201109-ab3d6c5b6470 ) require ( @@ -381,7 +382,6 @@ require ( go.opentelemetry.io/collector/connector v0.84.1-0.20230908201109-ab3d6c5b6470 // indirect go.opentelemetry.io/collector/extension/auth v0.84.1-0.20230908201109-ab3d6c5b6470 // indirect go.opentelemetry.io/collector/featuregate v1.0.0-rcv0014.0.20230908201109-ab3d6c5b6470 // indirect - go.opentelemetry.io/collector/semconv v0.84.1-0.20230908201109-ab3d6c5b6470 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.43.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.43.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.17.0 // indirect @@ -430,3 +430,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231020163023-8bdf732d320a diff --git a/go.sum b/go.sum index 4c64eb2f2c..d73acb8af3 100644 --- a/go.sum +++ b/go.sum @@ -145,36 +145,36 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713 h1:ACWoE8NqyI13oKdpg+074JIta5wmxsL1kOirTzDO5Gk= github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713/go.mod h1:4g9P7MZPReFnNJD0lpavI/LR0vIwlsTJov+hJoKT+nM= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20230928170322-0df38c533713 h1:wt44oIHsyO9tNSw36V2oWZG4cTzrpqBe7JKyfXHBa5w= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20230928170322-0df38c533713/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20230928170322-0df38c533713 h1:/DboazGzxalMCGykP//7s3m1+YwUaXydlUzX8uIrXUg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20230928170322-0df38c533713/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20230928170322-0df38c533713 h1:+vsrfX+HryDhdmXbbsopiLiTQd4UKQgBEQqZ2qZ/qGw= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20230928170322-0df38c533713/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20230928170322-0df38c533713 h1:7yTgx0xsW4p1JW0aiGI1B0swLJET933/28PCwWODTr0= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20230928170322-0df38c533713/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20230928170322-0df38c533713 h1:oaZq2zS/clGw4fl59WihfY3nRRzmI5gN+7ogDYK9uB4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20230928170322-0df38c533713/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20230928170322-0df38c533713 h1:5u5narTayNWsEqMAlUYcJ+mPWY0J0J00zXTVkpFLUwU= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20230928170322-0df38c533713/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20230928170322-0df38c533713 h1:py3+pCTK1NW0GGarcMnZEgY6J3JxTgKB9BpSQ90r39k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20230928170322-0df38c533713/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20230928170322-0df38c533713 h1:TrDLQ2BdMFt+bUp6dfzOFhG57AcY85h27UTei+jUdr0= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20230928170322-0df38c533713/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230928170322-0df38c533713 h1:N5zWIyEcfUr+t0EimIukJ0Z341YY0oew7ErHRJ1fnoc= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230928170322-0df38c533713/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a h1:E12KX+/EG0lgJLqs6j5egsqr+H3+CB8a/UaMnTVocbE= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a h1:Qf01P4PgCCtViETGGhBBtnyMGPpUssyZRV0m4zWyIzA= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a h1:ZrHh+eRCpcRKyQ7eyIf+OMDJ6zNj4cUCUH4HSLo7lVg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a h1:fzduYrFPdB/U4S2pr0oyo1dyLBwxtLcpnTPLw12ULjA= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a h1:JukQdbg+7mvU0kQ92j/KocsAYZ4x3ei85kEbEdTslPI= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a h1:g9MMmqMQXpy3HjJ64PoKtpaTSu61iv3SJNNAXQUmcdg= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a h1:U/fex+AoERTJeFo1IjDgsjjyKshIagmxu9c67GJp0j0= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a h1:RTebkRHNQMiZyvIR5U55deCMgEpH+Nerlhl6eeJ5dgI= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a h1:ad2h+5tqUTXFSdnJHGxO9UxUDpmr1r2n8Bcw/BYzfrw= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 h1:2daWNVtWNvRDoCTN5GG5N+LEM9OuY3RjJ0cboU3+xmM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20230928170322-0df38c533713 h1:SZf6K08K4cyoGDdkPthRGcsT1Gk6Z7o6QN01oM8Jr6A= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20230928170322-0df38c533713/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a h1:9xYr+Ru7GzJq9cobLqoV0DOp2XbZ+f6nuxo/wTOi4gQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713 h1:tKGat0aoXPkscaShYYRbnXH14asXqi1Iem4K3nMrNpk= github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713/go.mod h1:fbCDqcaNUzfvbpI4y91hT8UfV18VyIKfS42BsPRDAuc= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20230928170322-0df38c533713 h1:hDrhgnYst1yL2CLUmlwK1MqH4iqjLfxWZpw85X8cojE= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20230928170322-0df38c533713/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20230928170322-0df38c533713 h1:YPlfoYRq2+HxqibbpC3W+Go4WWJzVKEI5rziSHg/5Fc= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20230928170322-0df38c533713/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20230928170322-0df38c533713 h1:rAzBpcdCtVZqky+w3Nwv7tvTf7osX6WggT2/NEOHrkI= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20230928170322-0df38c533713/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a h1:ILOMnnQ8bUXg0Q9591LknoPqZJWbefDe3ktQpKv41E8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a h1:Yd5943Zx/1PDXzA9QUQ+duoxEfAqLa7U9fJGBuWK+Ic= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a h1:nI8fWWNGJ+/zsSjdrDC6iFHlFsFp1dCbj/NRZ6s9vUc= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= From 8bb1cbffe317320655f8dedc77d4c3eb48908d1b Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 20 Oct 2023 17:17:46 -0400 Subject: [PATCH 17/38] Add repo sync workflow (#587) --- .github/repo_sync_pr_template.md | 46 +++++++++++++++++++++++++ .github/workflows/repo-sync.yml | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 .github/repo_sync_pr_template.md create mode 100644 .github/workflows/repo-sync.yml diff --git a/.github/repo_sync_pr_template.md b/.github/repo_sync_pr_template.md new file mode 100644 index 0000000000..8d610babbc --- /dev/null +++ b/.github/repo_sync_pr_template.md @@ -0,0 +1,46 @@ +# Description of the issue +An automated PR to kickstart the process of syncing the latest changes from [cw-agent](https://github.com/aws/amazon-cloudwatch-agent/) + +# Description of changes + +### Follow the git CLI instructions resolve the merge conflicts + +```shell +git pull origin main +git checkout repo-sync-- +git merge main # do a regular merge -- we want to keep the commits +# resolve merge conflicts in your preferred IDE +git push -u origin repo-sync-- +``` + +Some useful commands +* [Restore conflict resolution in a single file](https://stackoverflow.com/questions/14409420/restart-undo-conflict-resolution-in-a-single-file) - `git checkout -m ` +* Total reset - `git merge --abort` + +### Related docs +* Resolving conflicts with: + * [Git CLI](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-using-the-command-line) + * [IntelliJ](https://www.jetbrains.com/help/idea/resolving-conflicts.html#distributed-version-control-systems) + * [GoLand](https://www.jetbrains.com/help/go/resolve-conflicts.html) + * [VSCode](https://learn.microsoft.com/en-us/visualstudio/version-control/git-resolve-conflicts?view=vs-2022) + +### Best practices + +* Remember to update all references from `amazon-cloudwatch-agent` to `private-amazon-cloudwatch-agent-staging` +* Resolve the `go.sum` with `go mod tidy`. Don't bother manually resolving conflicts in this file +* When finished, ensure builds work by using `make build` or `make release` +* When unsure or blocked, do a deep dive on the `git blame` for greater context. Maybe even look for the associated PR's and ask the original authors and PR approvers +* If another automated PR arrives before your work is merged, just close your current one and save the branch +* After your PR is approved, **do a regular merge to preserve the commits**. +* Remember to cleanup your commits because none of them will be squashed in a regular merge + +# License +By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. + +# Tests +n/a + +# Requirements +_Before commit the code, please do the following steps._ +1. Run `make fmt` and `make fmt-sh` +2. Run `make lint` \ No newline at end of file diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml new file mode 100644 index 0000000000..28bd6ea6bb --- /dev/null +++ b/.github/workflows/repo-sync.yml @@ -0,0 +1,58 @@ +# disable this workflow after beta phase +name: Sync with upstream + +on: + schedule: + - cron: "0 0 * * 0" # every sunday at 12AM + workflow_dispatch: + +env: + RUN_ID: "${{ github.run_number }}.${{ github.run_attempt }}" + UPSTREAM: "https://github.com/aws/amazon-cloudwatch-agent.git" + +jobs: + # gets the last commit hash from public/master and defines the branch name + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idoutputs + define-branch-name: + runs-on: ubuntu-latest + steps: + - name: Get last commit hash from public + id: get-last-commit + run: echo "hash=$(git ls-remote ${{ env.UPSTREAM }} HEAD | awk '{print $1;}')" >> $GITHUB_OUTPUT + outputs: + LAST_COMMIT: ${{ steps.get-last-commit.outputs.hash }} + PR_BRANCH: "repo-sync-${{ steps.get-last-commit.outputs.hash }}-${{ env.RUN_ID }}" + + # pushes the latest from public/main to private/repo-sync + # https://github.com/marketplace/actions/github-repo-sync + create-branch: + needs: define-branch-name + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + - name: repo-sync + uses: repo-sync/github-sync@v2 + with: + source_repo: ${{ env.UPSTREAM }} + source_branch: "main" + destination_branch: ${{ needs.define-branch-name.outputs.PR_BRANCH }} + github_token: ${{ secrets.WILLIAZZ_PAT }} + + # upon create-branch completion, creates a PR from private/repo-sync to private/main + # https://github.com/marketplace/actions/github-pull-request-action + create-pr: + needs: [define-branch-name, create-branch] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: pull-request + uses: repo-sync/pull-request@v2 + with: + source_branch: ${{ needs.define-branch-name.outputs.PR_BRANCH }} + destination_branch: "main-apm" + github_token: ${{ secrets.GITHUB_TOKEN }} + pr_title: "Automated sync with upstream - last commit ${{ needs.define-branch-name.outputs.LAST_COMMIT }} - run #${{ env.RUN_ID }}" + pr_template: ".github/repo_sync_pr_template.md" + pr_allow_empty: false \ No newline at end of file From 76214cc09c836475f626663bb2dace769edda4a0 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Mon, 23 Oct 2023 10:09:09 -0400 Subject: [PATCH 18/38] fix unit test --- .../tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml index 648a4abaeb..cf5596a2eb 100644 --- a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml @@ -127,7 +127,7 @@ exporters: dimension_rollup_option: NoDimensionRollup disable_metric_extraction: true eks_fargate_container_insights_enabled: false - endpoint: "" + endpoint: "https://fake_endpoint" enhanced_container_insights: false imds_retries: 1 local_mode: false From aebc128b0f575a065a90e34bade3530e2a5793e0 Mon Sep 17 00:00:00 2001 From: Dinakar Chappa Date: Mon, 23 Oct 2023 16:05:25 -0400 Subject: [PATCH 19/38] EMF Exporter does not forward metrics if IRSA is enabled (#915) --- plugins/inputs/logfile/logfile_test.go | 2 +- translator/translate/otel/exporter/awsemf/translator.go | 1 + translator/translate/otel/exporter/awsemf/translator_test.go | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/logfile/logfile_test.go b/plugins/inputs/logfile/logfile_test.go index f5a38ab48c..2c1a2e98dd 100644 --- a/plugins/inputs/logfile/logfile_test.go +++ b/plugins/inputs/logfile/logfile_test.go @@ -453,7 +453,7 @@ func createWriteRead(t *testing.T, prefix string, logFile *LogFile, done chan bo select { case <-done2: t.Log("Child completed before timeout (as expected)") - case <-time.After(time.Second * 10): + case <-time.After(time.Second * 20): require.Fail(t, "timeout waiting for child") } t.Log("Verify 1st temp file was auto deleted.") diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index c52a79f909..b87ca44d60 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -105,6 +105,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if credentialsFileKey, ok := agent.Global_Config.Credentials[agent.CredentialsFile_Key]; ok { cfg.AWSSessionSettings.SharedCredentialsFile = []string{fmt.Sprintf("%v", credentialsFileKey)} } + cfg.AWSSessionSettings.RoleARN = agent.Global_Config.Role_arn cfg.AWSSessionSettings.IMDSRetries = retryer.GetDefaultRetryNumber() if isEcs(c) { diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index b2833c342d..a0d264647f 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -16,6 +16,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" legacytranslator "github.com/aws/amazon-cloudwatch-agent/translator" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) @@ -24,6 +25,8 @@ var nilMetricDescriptorsSlice []awsemfexporter.MetricDescriptor func TestTranslator(t *testing.T) { tt := NewTranslator() + agent.Global_Config.Region = "us-east-1" + agent.Global_Config.Role_arn = "global_arn" require.EqualValues(t, "awsemf", tt.ID().String()) testCases := map[string]struct { env map[string]string @@ -680,6 +683,8 @@ func TestTranslator(t *testing.T) { require.Equal(t, testCase.want["resource_to_telemetry_conversion"], gotCfg.ResourceToTelemetrySettings) require.ElementsMatch(t, testCase.want["metric_declarations"], gotCfg.MetricDeclarations) require.ElementsMatch(t, testCase.want["metric_descriptors"], gotCfg.MetricDescriptors) + require.Equal(t, "global_arn", gotCfg.RoleARN) + require.Equal(t, "us-east-1", gotCfg.Region) } }) } From b06be951c80383682a4af61a0d50ead97aa5988b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 11:21:32 -0400 Subject: [PATCH 20/38] Update OTel fork components to f645697bf35099724d59a5d4fbeebb1594ce7a6e (#592) Co-authored-by: Github Action --- go.mod | 41 ++++++++++++++-------------- go.sum | 85 +++++++++++++++++++++++++++++++++------------------------- 2 files changed, 69 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index 1acb0b1ddc..287ba0f719 100644 --- a/go.mod +++ b/go.mod @@ -6,34 +6,34 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2022 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231023230448-f645697bf350 // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231023230448-f645697bf350 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231023230448-f645697bf350 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231023230448-f645697bf350 -replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a +replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231023230448-f645697bf350 replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713 @@ -89,8 +89,8 @@ replace github.com/openshift/api v3.9.0+incompatible => github.com/openshift/api require ( github.com/BurntSushi/toml v1.3.2 github.com/Jeffail/gabs v1.4.0 - github.com/aws/aws-sdk-go v1.45.2 - github.com/aws/aws-sdk-go-v2 v1.19.0 + github.com/aws/aws-sdk-go v1.45.24 + github.com/aws/aws-sdk-go-v2 v1.21.2 github.com/aws/aws-sdk-go-v2/config v1.18.25 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.10 @@ -99,7 +99,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecs v1.28.1 github.com/aws/aws-sdk-go-v2/service/efs v1.19.7 github.com/aws/aws-sdk-go-v2/service/eks v1.27.15 - github.com/aws/smithy-go v1.13.5 + github.com/aws/smithy-go v1.15.0 github.com/bigkevmcd/go-configparser v0.0.0-20200217161103-d137835d2579 github.com/go-kit/log v0.2.1 github.com/gobwas/glob v0.2.3 @@ -191,6 +191,7 @@ require ( github.com/alecthomas/participle v0.4.1 // indirect github.com/alecthomas/participle/v2 v2.0.0 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20231023152757-c6e2437e6590 // indirect github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230928170322-0df38c533713 // indirect github.com/antchfx/jsonquery v1.1.5 // indirect github.com/antchfx/xmlquery v1.3.9 // indirect @@ -200,10 +201,10 @@ require ( github.com/apache/thrift v0.16.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect @@ -434,4 +435,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231020163023-8bdf732d320a +replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231023230448-f645697bf350 diff --git a/go.sum b/go.sum index 22c05e4767..9b4d8c96ed 100644 --- a/go.sum +++ b/go.sum @@ -146,36 +146,38 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713 h1:ACWoE8NqyI13oKdpg+074JIta5wmxsL1kOirTzDO5Gk= github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713/go.mod h1:4g9P7MZPReFnNJD0lpavI/LR0vIwlsTJov+hJoKT+nM= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a h1:E12KX+/EG0lgJLqs6j5egsqr+H3+CB8a/UaMnTVocbE= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a h1:Qf01P4PgCCtViETGGhBBtnyMGPpUssyZRV0m4zWyIzA= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a h1:ZrHh+eRCpcRKyQ7eyIf+OMDJ6zNj4cUCUH4HSLo7lVg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a h1:fzduYrFPdB/U4S2pr0oyo1dyLBwxtLcpnTPLw12ULjA= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a h1:JukQdbg+7mvU0kQ92j/KocsAYZ4x3ei85kEbEdTslPI= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a h1:g9MMmqMQXpy3HjJ64PoKtpaTSu61iv3SJNNAXQUmcdg= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a h1:U/fex+AoERTJeFo1IjDgsjjyKshIagmxu9c67GJp0j0= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a h1:RTebkRHNQMiZyvIR5U55deCMgEpH+Nerlhl6eeJ5dgI= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a h1:ad2h+5tqUTXFSdnJHGxO9UxUDpmr1r2n8Bcw/BYzfrw= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231023230448-f645697bf350 h1:9dbD1NtMVxgOaXRRLlEEXVkQ/79Xamg3VmpwZXwCqhI= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231023230448-f645697bf350/go.mod h1:/8w8sPrpOeADRJgMsu8o4jOiFX29zCC899+ao7S1GXI= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231023230448-f645697bf350 h1:shavMGqlJ1U0E3zKWGmatc3S629dx2Eg6O7i/zDSv+U= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231023230448-f645697bf350/go.mod h1:UAXcRSojI8I0Kb9iS9a2v7J/iPrQ1loJIsBprSaVdFo= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231023230448-f645697bf350 h1:Exv0zXBGFyISCsTATWtN7kjJ8rwTNOlj8Exvp/56qRE= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231023230448-f645697bf350/go.mod h1:cr4dmBlfnMVYT+gyKUAKh39zQu5u/UAukxQj15MdZ18= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20231023152757-c6e2437e6590 h1:uUCPnX2C5C36iyX46N1eUjmp4LRpSTGeexgUWmohv7c= +github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20231023152757-c6e2437e6590/go.mod h1:uOQa5/9Jle9VADEdWCXL4AbJr35NJQil30tapcTHQlw= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231023230448-f645697bf350 h1:rUW/xq+BtDoVpvweywhEWF9Ahff3ea8UPHP1Z57xI/o= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231023230448-f645697bf350/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231023230448-f645697bf350 h1:ElTqozKQqpF8rZoEUc5sO7yVtEJ7QGdSakcXL3BDl+s= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231023230448-f645697bf350/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231023230448-f645697bf350 h1:X7Hs9ak6cPFayuxQM5DAq11Qzhx+J6KTstqYr0dN1yE= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231023230448-f645697bf350/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231023230448-f645697bf350 h1:EuCEm4+sBiyeYNl4yeuPwPCFKxKad/insaB2vtOztpw= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231023230448-f645697bf350/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231023230448-f645697bf350 h1:7VjZoihy0KbhCSx3LUW+w8hIVH9HBFr7wpkotUm6pGw= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231023230448-f645697bf350/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231023230448-f645697bf350 h1:IvSTtrtjzTL6Khw9H4V+wAqGGhPpUY6N2mG3YKLE8FM= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231023230448-f645697bf350/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 h1:2daWNVtWNvRDoCTN5GG5N+LEM9OuY3RjJ0cboU3+xmM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a h1:9xYr+Ru7GzJq9cobLqoV0DOp2XbZ+f6nuxo/wTOi4gQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231023230448-f645697bf350 h1:ZuWsd6s1Nq6gLRQXLt1TxIJVFOoG9F/LQlNgg7JfBaQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231023230448-f645697bf350/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713 h1:tKGat0aoXPkscaShYYRbnXH14asXqi1Iem4K3nMrNpk= github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713/go.mod h1:fbCDqcaNUzfvbpI4y91hT8UfV18VyIKfS42BsPRDAuc= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a h1:ILOMnnQ8bUXg0Q9591LknoPqZJWbefDe3ktQpKv41E8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a h1:Yd5943Zx/1PDXzA9QUQ+duoxEfAqLa7U9fJGBuWK+Ic= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a h1:nI8fWWNGJ+/zsSjdrDC6iFHlFsFp1dCbj/NRZ6s9vUc= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231023230448-f645697bf350 h1:xtFHI743pFqd0LRXqWTxG+uB/xC+IIAAgfJGVD+Bceo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231023230448-f645697bf350/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231023230448-f645697bf350 h1:ctkNAXD6VRLwSS7tyFTwnA/v2LtlFtu7o7/6vXMWG9A= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231023230448-f645697bf350/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231023230448-f645697bf350 h1:SwoVPsXDPeF3pSqjXa4V5vTArwqJVLaqVBnio5/15xo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231023230448-f645697bf350/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= @@ -206,16 +208,18 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.45.2 h1:hTong9YUklQKqzrGk3WnKABReb5R8GjbG4Y6dEQfjnk= -github.com/aws/aws-sdk-go v1.45.2/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.45.24 h1:TZx/CizkmCQn8Rtsb11iLYutEQVGK5PK9wAhwouELBo= +github.com/aws/aws-sdk-go v1.45.24/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.2.0 h1:scBthy70MB3m4LCMFaBcmYCyR2XWOz6MxSfdSu/+fQo= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= github.com/aws/aws-sdk-go-v2/config v1.18.25 h1:JuYyZcnMPBiFqn87L2cRppo+rNwgah6YwD3VuyvaW6Q= github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= @@ -229,16 +233,19 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.3 h1:0O72494cCsazjpsGfo+LXe github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.10 h1:moHEk4wbdc8VNvff4UOLuXVHtjh7YtsGdiyB0MrPPKg= github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.10/go.mod h1:P3qp1VYVoxHgDhpDDCTre1ee9IKpmgqnUoOb+8RA9qI= @@ -255,14 +262,16 @@ github.com/aws/aws-sdk-go-v2/service/efs v1.19.7 h1:BmyhflgczNmmuAPFhAhMQuLc9zSH github.com/aws/aws-sdk-go-v2/service/efs v1.19.7/go.mod h1:ENSgtHyPiYyBcTAi26Hpr8Xp636IB18qr0D5Ho8EQWA= github.com/aws/aws-sdk-go-v2/service/eks v1.27.15 h1:Q48ivwZJ136hfkk8Dua1fMM7m1e1s/0rBRyRX/J9XAY= github.com/aws/aws-sdk-go-v2/service/eks v1.27.15/go.mod h1:9mqDBj08MtFxKFQWUEMm4iFnIdM9gFpnSJvHUEIfsiU= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.8.0 h1:wS94St7YDmLhrPJw3mjJfCfHHOABS3G9c//mDZRzELU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.6.0 h1:q/O6wGx7MFwWfRNgTIVmGgXGBz9UKv16eSX1uuWdM7A= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.1 h1:YEz2KMyqK2zyG3uOa0l2xBc/H6NUVJir8FhwHQHF3rc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= github.com/aws/aws-sdk-go-v2/service/kinesis v1.13.0 h1:wqLvwC4qdrrGikudu8Z9X2sb79BYUYWAgMF5BGFQJY8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.16.0 h1:dt1JQFj/135ozwGIWeCM3aQ8N/kB3Xu3Uu4r9zuOIyc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0 h1:wl5dxN1NONhTDQD9uaEvNsDRX29cBmGED/nl0jkWlt4= github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 h1:UBQjaMTCKwyUYwiVnUt6toEJwGXsLBI6al083tpjJzY= github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= @@ -273,8 +282,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 h1:2DQLAKDteoEDI8zpCzqBMaZlJuoE github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.3.2 h1:1s/RRA5Owuz4/G/eWCdCKgC+9zaz2vxFsRSwe7R3cPY= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/telegraf v0.10.2-0.20220502160831-c20ebe67c5ef h1:O53nKbZm2XpdudUywNdqbohwUxje9k4vE0xRXWeIVbE= github.com/aws/telegraf v0.10.2-0.20220502160831-c20ebe67c5ef/go.mod h1:6maU8S0L0iMSa0ZvH5b2W7dBX1xjK0D5ONAqe7WTqXc= github.com/aws/telegraf/patches/gopsutil/v3 v3.0.0-20230915153624-7629361f8380 h1:LyWVxYjlmdI9ruL66nvr85SRmUA7sScaTNEAHgbsEHc= From ad8903e5ccde8ea27a1f12d8059a7a72b7189c4a Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Wed, 25 Oct 2023 14:00:25 -0400 Subject: [PATCH 21/38] Add support for AppSignals --- .../processors/awsappsignals}/README.md | 12 ++--- .../processors/awsappsignals}/config.go | 4 +- .../processors/awsappsignals}/config_test.go | 6 +-- .../customconfiguration/common.go | 0 .../customconfiguration/dropper.go | 0 .../customconfiguration/dropper_test.go | 0 .../customconfiguration/keeper.go | 0 .../customconfiguration/keeper_test.go | 0 .../customconfiguration/replacer.go | 0 .../customconfiguration/replacer_test.go | 0 .../processors/awsappsignals}/factory.go | 10 ++-- .../normalizer/attributesnormalizer.go | 0 .../normalizer/attributesnormalizer_test.go | 0 .../internal/resolver/attributesresolver.go | 0 .../resolver/attributesresolver_test.go | 0 .../awsappsignals}/internal/resolver/eks.go | 0 .../internal/resolver/eks_test.go | 0 .../awsappsignals}/internal/resolver/types.go | 0 .../processors/awsappsignals}/processor.go | 20 +++---- .../awsappsignals}/testdata/config.yaml | 2 +- service/defaultcomponents/components.go | 4 +- service/defaultcomponents/components_test.go | 2 +- translator/config/schema.json | 4 +- ... => appsignals_and_kubernetes_config.json} | 4 +- ... => appsignals_and_kubernetes_config.yaml} | 14 ++--- .../sampleConfig/base_apm_config.json | 4 +- .../sampleConfig/base_apm_config.yaml | 32 ++++++------ translator/tocwconfig/tocwconfig_test.go | 14 ++--- translator/translate/otel/common/apm.go | 2 +- translator/translate/otel/common/apm_test.go | 6 +-- translator/translate/otel/common/common.go | 14 ++--- ...ig_eks.yaml => appsignals_config_eks.yaml} | 4 +- ...ic.yaml => appsignals_config_generic.yaml} | 4 +- .../otel/exporter/awsemf/translator.go | 16 +++--- .../otel/exporter/awsemf/translator_test.go | 16 +++--- .../otel/exporter/awsxray/translator.go | 6 +-- .../otel/exporter/awsxray/translator_test.go | 4 +- .../{apm => appsignals}/translator.go | 18 +++---- .../{apm => appsignals}/translator_test.go | 32 ++++++------ .../testdata/config_eks.yaml | 0 .../testdata/config_generic.yaml | 0 .../testdata/invalidRulesConfig.json | 2 +- .../testdata/validRulesConfig.json | 2 +- .../testdata/validRulesConfigEKS.yaml | 0 .../testdata/validRulesConfigGeneric.yaml | 0 .../{awsapm => awsappsignals}/translator.go | 18 +++---- .../translator_test.go | 52 +++++++++---------- .../resourcedetection/translator_test.go | 4 +- ...apm_config.yaml => appsignals_config.yaml} | 0 .../otel/receiver/otlp/translator.go | 8 +-- .../otel/receiver/otlp/translator_test.go | 8 +-- translator/translate/otel/translate_otel.go | 6 +-- .../translate/otel/translate_otel_test.go | 14 ++--- 53 files changed, 184 insertions(+), 184 deletions(-) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/README.md (90%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/config.go (76%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/config_test.go (93%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/customconfiguration/common.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/customconfiguration/dropper.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/customconfiguration/dropper_test.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/customconfiguration/keeper.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/customconfiguration/keeper_test.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/customconfiguration/replacer.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/customconfiguration/replacer_test.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/factory.go (90%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/internal/normalizer/attributesnormalizer.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/internal/normalizer/attributesnormalizer_test.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/internal/resolver/attributesresolver.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/internal/resolver/attributesresolver_test.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/internal/resolver/eks.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/internal/resolver/eks_test.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/internal/resolver/types.go (100%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/processor.go (90%) rename {processor/awsapmprocessor => plugins/processors/awsappsignals}/testdata/config.yaml (97%) rename translator/tocwconfig/sampleConfig/{apm_and_kubernetes_config.json => appsignals_and_kubernetes_config.json} (90%) rename translator/tocwconfig/sampleConfig/{apm_and_kubernetes_config.yaml => appsignals_and_kubernetes_config.yaml} (98%) rename translator/translate/otel/exporter/awsemf/{apm_config_eks.yaml => appsignals_config_eks.yaml} (96%) rename translator/translate/otel/exporter/awsemf/{apm_config_generic.yaml => appsignals_config_generic.yaml} (93%) rename translator/translate/otel/pipeline/{apm => appsignals}/translator.go (85%) rename translator/translate/otel/pipeline/{apm => appsignals}/translator_test.go (79%) rename translator/translate/otel/processor/{awsapm => awsappsignals}/testdata/config_eks.yaml (100%) rename translator/translate/otel/processor/{awsapm => awsappsignals}/testdata/config_generic.yaml (100%) rename translator/translate/otel/processor/{awsapm => awsappsignals}/testdata/invalidRulesConfig.json (93%) rename translator/translate/otel/processor/{awsapm => awsappsignals}/testdata/validRulesConfig.json (98%) rename translator/translate/otel/processor/{awsapm => awsappsignals}/testdata/validRulesConfigEKS.yaml (100%) rename translator/translate/otel/processor/{awsapm => awsappsignals}/testdata/validRulesConfigGeneric.yaml (100%) rename translator/translate/otel/processor/{awsapm => awsappsignals}/translator.go (85%) rename translator/translate/otel/processor/{awsapm => awsappsignals}/translator_test.go (59%) rename translator/translate/otel/receiver/otlp/{apm_config.yaml => appsignals_config.yaml} (100%) diff --git a/processor/awsapmprocessor/README.md b/plugins/processors/awsappsignals/README.md similarity index 90% rename from processor/awsapmprocessor/README.md rename to plugins/processors/awsappsignals/README.md index e380b3b912..f270a3a734 100644 --- a/processor/awsapmprocessor/README.md +++ b/plugins/processors/awsappsignals/README.md @@ -1,6 +1,6 @@ -# AWS APM Processor for Amazon Cloudwatch Agent +# AWS AppSignals Processor for Amazon Cloudwatch Agent -The AWS APM processor is used to reduce the cardinality of telemtry metrics and traces before exporting them to CloudWatch Logs via [EMF](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awsemfexporter) and [X-Ray](github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter) respectively. +The AWS AppSignals processor is used to reduce the cardinality of telemtry metrics and traces before exporting them to CloudWatch Logs via [EMF](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awsemfexporter) and [X-Ray](github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter) respectively. It reduces the cardinality of metrics/traces via 3 types of actions, `keep`, `drop` and `replace`, which are configured by users. CloudWatch Agent(CWA) customers will configure these rules with their CWA configurations, which will get translated to yaml. Note: Traces support only `replace` actions and are implicitly pulled from the logs section of the CWA configuration @@ -47,10 +47,10 @@ A replacements section defines a matching against the dimensions of incoming met | `value` | Value to replace current dimension value with | "" | -## AWS APM Processor Configuration Example +## AWS AppSignals Processor Configuration Example ```yaml -awsapm: +awsappsignals: resolvers: ["eks"] rules: - selectors: @@ -92,12 +92,12 @@ awsapm: }, "traces": { "traces_collected": { - "apm": {} + "app_signals": {} } }, "logs": { "metrics_collected": { - "apm": { + "app_signals": { "rules": [ { "selectors": [ diff --git a/processor/awsapmprocessor/config.go b/plugins/processors/awsappsignals/config.go similarity index 76% rename from processor/awsapmprocessor/config.go rename to plugins/processors/awsappsignals/config.go index 19572e01cf..dfc01ec00a 100644 --- a/processor/awsapmprocessor/config.go +++ b/plugins/processors/awsappsignals/config.go @@ -1,10 +1,10 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsapmprocessor +package awsappsignals import ( - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/customconfiguration" ) type Config struct { diff --git a/processor/awsapmprocessor/config_test.go b/plugins/processors/awsappsignals/config_test.go similarity index 93% rename from processor/awsapmprocessor/config_test.go rename to plugins/processors/awsappsignals/config_test.go index 36c0e142c5..02ff05f91d 100644 --- a/processor/awsapmprocessor/config_test.go +++ b/plugins/processors/awsappsignals/config_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsapmprocessor +package awsappsignals import ( "path/filepath" @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap/confmaptest" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/customconfiguration" ) func TestLoadConfig(t *testing.T) { @@ -24,7 +24,7 @@ func TestLoadConfig(t *testing.T) { errorMessage string }{ { - id: component.NewIDWithName("awsapm", ""), + id: component.NewIDWithName("awsappsignals", ""), expected: &Config{ Resolvers: []string{"eks"}, Rules: []customconfiguration.Rule{ diff --git a/processor/awsapmprocessor/customconfiguration/common.go b/plugins/processors/awsappsignals/customconfiguration/common.go similarity index 100% rename from processor/awsapmprocessor/customconfiguration/common.go rename to plugins/processors/awsappsignals/customconfiguration/common.go diff --git a/processor/awsapmprocessor/customconfiguration/dropper.go b/plugins/processors/awsappsignals/customconfiguration/dropper.go similarity index 100% rename from processor/awsapmprocessor/customconfiguration/dropper.go rename to plugins/processors/awsappsignals/customconfiguration/dropper.go diff --git a/processor/awsapmprocessor/customconfiguration/dropper_test.go b/plugins/processors/awsappsignals/customconfiguration/dropper_test.go similarity index 100% rename from processor/awsapmprocessor/customconfiguration/dropper_test.go rename to plugins/processors/awsappsignals/customconfiguration/dropper_test.go diff --git a/processor/awsapmprocessor/customconfiguration/keeper.go b/plugins/processors/awsappsignals/customconfiguration/keeper.go similarity index 100% rename from processor/awsapmprocessor/customconfiguration/keeper.go rename to plugins/processors/awsappsignals/customconfiguration/keeper.go diff --git a/processor/awsapmprocessor/customconfiguration/keeper_test.go b/plugins/processors/awsappsignals/customconfiguration/keeper_test.go similarity index 100% rename from processor/awsapmprocessor/customconfiguration/keeper_test.go rename to plugins/processors/awsappsignals/customconfiguration/keeper_test.go diff --git a/processor/awsapmprocessor/customconfiguration/replacer.go b/plugins/processors/awsappsignals/customconfiguration/replacer.go similarity index 100% rename from processor/awsapmprocessor/customconfiguration/replacer.go rename to plugins/processors/awsappsignals/customconfiguration/replacer.go diff --git a/processor/awsapmprocessor/customconfiguration/replacer_test.go b/plugins/processors/awsappsignals/customconfiguration/replacer_test.go similarity index 100% rename from processor/awsapmprocessor/customconfiguration/replacer_test.go rename to plugins/processors/awsappsignals/customconfiguration/replacer_test.go diff --git a/processor/awsapmprocessor/factory.go b/plugins/processors/awsappsignals/factory.go similarity index 90% rename from processor/awsapmprocessor/factory.go rename to plugins/processors/awsappsignals/factory.go index c91290885d..c00150a85a 100644 --- a/processor/awsapmprocessor/factory.go +++ b/plugins/processors/awsappsignals/factory.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsapmprocessor +package awsappsignals import ( "context" @@ -15,7 +15,7 @@ import ( const ( // The value of "type" key in configuration. - typeStr = "awsapm" + typeStr = "awsappsignals" // The stability level of the processor. stability = component.StabilityLevelBeta ) @@ -85,12 +85,12 @@ func createMetricsProcessor( func createProcessor( params processor.CreateSettings, cfg component.Config, -) (*awsapmprocessor, error) { +) (*awsappsignalsprocessor, error) { pCfg, ok := cfg.(*Config) if !ok { - return nil, errors.New("could not initialize awsapmprocessor") + return nil, errors.New("could not initialize awsappsignalsprocessor") } - ap := &awsapmprocessor{logger: params.Logger, config: pCfg} + ap := &awsappsignalsprocessor{logger: params.Logger, config: pCfg} return ap, nil } diff --git a/processor/awsapmprocessor/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go similarity index 100% rename from processor/awsapmprocessor/internal/normalizer/attributesnormalizer.go rename to plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go diff --git a/processor/awsapmprocessor/internal/normalizer/attributesnormalizer_test.go b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go similarity index 100% rename from processor/awsapmprocessor/internal/normalizer/attributesnormalizer_test.go rename to plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer_test.go diff --git a/processor/awsapmprocessor/internal/resolver/attributesresolver.go b/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go similarity index 100% rename from processor/awsapmprocessor/internal/resolver/attributesresolver.go rename to plugins/processors/awsappsignals/internal/resolver/attributesresolver.go diff --git a/processor/awsapmprocessor/internal/resolver/attributesresolver_test.go b/plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go similarity index 100% rename from processor/awsapmprocessor/internal/resolver/attributesresolver_test.go rename to plugins/processors/awsappsignals/internal/resolver/attributesresolver_test.go diff --git a/processor/awsapmprocessor/internal/resolver/eks.go b/plugins/processors/awsappsignals/internal/resolver/eks.go similarity index 100% rename from processor/awsapmprocessor/internal/resolver/eks.go rename to plugins/processors/awsappsignals/internal/resolver/eks.go diff --git a/processor/awsapmprocessor/internal/resolver/eks_test.go b/plugins/processors/awsappsignals/internal/resolver/eks_test.go similarity index 100% rename from processor/awsapmprocessor/internal/resolver/eks_test.go rename to plugins/processors/awsappsignals/internal/resolver/eks_test.go diff --git a/processor/awsapmprocessor/internal/resolver/types.go b/plugins/processors/awsappsignals/internal/resolver/types.go similarity index 100% rename from processor/awsapmprocessor/internal/resolver/types.go rename to plugins/processors/awsappsignals/internal/resolver/types.go diff --git a/processor/awsapmprocessor/processor.go b/plugins/processors/awsappsignals/processor.go similarity index 90% rename from processor/awsapmprocessor/processor.go rename to plugins/processors/awsappsignals/processor.go index 18b37fe682..b4375dedc2 100644 --- a/processor/awsapmprocessor/processor.go +++ b/plugins/processors/awsappsignals/processor.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsapmprocessor +package awsappsignals import ( "context" @@ -12,9 +12,9 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/customconfiguration" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/normalizer" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/internal/resolver" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/normalizer" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/resolver" ) const ( @@ -35,7 +35,7 @@ type stopper interface { Stop(context.Context) error } -type awsapmprocessor struct { +type awsappsignalsprocessor struct { logger *zap.Logger config *Config customReplacer *customconfiguration.ReplaceActions @@ -45,7 +45,7 @@ type awsapmprocessor struct { stoppers []stopper } -func (ap *awsapmprocessor) Start(_ context.Context, _ component.Host) error { +func (ap *awsappsignalsprocessor) Start(_ context.Context, _ component.Host) error { attributesResolver := resolver.NewAttributesResolver(ap.config.Resolvers, ap.logger) ap.stoppers = append(ap.stoppers, attributesResolver) ap.metricMutators = append(ap.metricMutators, attributesResolver) @@ -64,7 +64,7 @@ func (ap *awsapmprocessor) Start(_ context.Context, _ component.Host) error { return nil } -func (ap *awsapmprocessor) Shutdown(ctx context.Context) error { +func (ap *awsappsignalsprocessor) Shutdown(ctx context.Context) error { for _, stopper := range ap.stoppers { err := stopper.Stop(ctx) if err != nil { @@ -74,7 +74,7 @@ func (ap *awsapmprocessor) Shutdown(ctx context.Context) error { return nil } -func (ap *awsapmprocessor) processTraces(ctx context.Context, td ptrace.Traces) (ptrace.Traces, error) { +func (ap *awsappsignalsprocessor) processTraces(ctx context.Context, td ptrace.Traces) (ptrace.Traces, error) { rss := td.ResourceSpans() for i := 0; i < rss.Len(); i++ { rs := rss.At(i) @@ -97,7 +97,7 @@ func (ap *awsapmprocessor) processTraces(ctx context.Context, td ptrace.Traces) return td, nil } -func (ap *awsapmprocessor) processMetrics(ctx context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { +func (ap *awsappsignalsprocessor) processMetrics(ctx context.Context, md pmetric.Metrics) (pmetric.Metrics, error) { rms := md.ResourceMetrics() for i := 0; i < rms.Len(); i++ { rs := rms.At(i) @@ -117,7 +117,7 @@ func (ap *awsapmprocessor) processMetrics(ctx context.Context, md pmetric.Metric // Attributes are provided for each log and trace, but not at the metric level // Need to process attributes for every data point within a metric. -func (ap *awsapmprocessor) processMetricAttributes(ctx context.Context, m pmetric.Metric, resourceAttribes pcommon.Map) { +func (ap *awsappsignalsprocessor) processMetricAttributes(ctx context.Context, m pmetric.Metric, resourceAttribes pcommon.Map) { // This is a lot of repeated code, but since there is no single parent superclass // between metric data types, we can't use polymorphism. diff --git a/processor/awsapmprocessor/testdata/config.yaml b/plugins/processors/awsappsignals/testdata/config.yaml similarity index 97% rename from processor/awsapmprocessor/testdata/config.yaml rename to plugins/processors/awsappsignals/testdata/config.yaml index d3b1ae6509..71c5c84e97 100644 --- a/processor/awsapmprocessor/testdata/config.yaml +++ b/plugins/processors/awsappsignals/testdata/config.yaml @@ -1,4 +1,4 @@ -awsapm: +awsappsignals: resolvers: [eks] rules: - selectors: diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index 3afe64f731..cd53e4d4b0 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -27,7 +27,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/plugins/outputs/cloudwatch" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" ) func Factories() (otelcol.Factories, error) { @@ -45,7 +45,7 @@ func Factories() (otelcol.Factories, error) { } if factories.Processors, err = processor.MakeFactoryMap( - awsapmprocessor.NewFactory(), + awsappsignals.NewFactory(), batchprocessor.NewFactory(), cumulativetodeltaprocessor.NewFactory(), ec2tagger.NewFactory(), diff --git a/service/defaultcomponents/components_test.go b/service/defaultcomponents/components_test.go index a555a77852..47117692e2 100644 --- a/service/defaultcomponents/components_test.go +++ b/service/defaultcomponents/components_test.go @@ -29,7 +29,7 @@ func TestComponents(t *testing.T) { processors := factories.Processors assert.Len(t, processors, processorCount) - assert.NotNil(t, processors["awsapm"]) + assert.NotNil(t, processors["awsappsignals"]) assert.NotNil(t, processors["batch"]) assert.NotNil(t, processors["cumulativetodelta"]) assert.NotNil(t, processors["ec2tagger"]) diff --git a/translator/config/schema.json b/translator/config/schema.json index 97ab8003ec..d2854942df 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -511,7 +511,7 @@ "metrics_collected": { "type": "object", "properties": { - "apm": { + "app_signals": { "type": "object", "properties": { "rules": { @@ -888,7 +888,7 @@ "traces_collected": { "type": "object", "properties": { - "apm": { + "app_signals": { "type": "object", "properties": {}, "additionalProperties": true diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.json b/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.json similarity index 90% rename from translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.json rename to translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.json index d30415469f..9585df211b 100644 --- a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.json +++ b/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.json @@ -4,7 +4,7 @@ }, "logs": { "metrics_collected": { - "apm": { + "app_signals": { }, "kubernetes": { "cluster_name": "TestCluster", @@ -18,7 +18,7 @@ }, "traces": { "traces_collected": { - "apm": {} + "app_signals": {} } } } \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml similarity index 98% rename from translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml rename to translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml index cf5596a2eb..031c9b7580 100644 --- a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml @@ -1,6 +1,6 @@ connectors: {} exporters: - awsemf/apm: + awsemf/appsignals: certificate_file_path: "" detailed_metrics: false dimension_rollup_option: NoDimensionRollup @@ -10,7 +10,7 @@ exporters: enhanced_container_insights: false imds_retries: 0 local_mode: false - log_group_name: /aws/apm/eks + log_group_name: /aws/appsignals/eks log_retention: 0 log_stream_name: "" max_retries: 2 @@ -105,7 +105,7 @@ exporters: - Fault - Error metric_descriptors: [] - namespace: AWS/APM + namespace: AppSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -240,7 +240,7 @@ exporters: role_arn: "" shared_credentials_file: [] version: "0" - awsxray/apm: + awsxray/appsignals: aws_log_groups: [] certificate_file_path: "" endpoint: "" @@ -271,7 +271,7 @@ exporters: enabled: true include_metadata: true extensions: - awsproxy/apm: + awsproxy/appsignals: aws_endpoint: "" endpoint: 0.0.0.0:2000 local_mode: false @@ -279,7 +279,7 @@ extensions: region: "" role_arn: "" processors: - awsapm: + awsappsignals: resolvers: - eks rules: [] @@ -575,7 +575,7 @@ receivers: resource_arn: "" role_arn: "" shared_credentials_file: [] - otlp/apm: + otlp/appsignals: protocols: grpc: auth: null diff --git a/translator/tocwconfig/sampleConfig/base_apm_config.json b/translator/tocwconfig/sampleConfig/base_apm_config.json index 2d4daf5fe2..255feae87a 100644 --- a/translator/tocwconfig/sampleConfig/base_apm_config.json +++ b/translator/tocwconfig/sampleConfig/base_apm_config.json @@ -4,13 +4,13 @@ }, "logs": { "metrics_collected": { - "apm": {} + "app_signals": {} }, "endpoint_override":"https://fake_endpoint" }, "traces": { "traces_collected": { - "apm": {} + "app_signals": {} }, "endpoint_override":"https://fake_endpoint" } diff --git a/translator/tocwconfig/sampleConfig/base_apm_config.yaml b/translator/tocwconfig/sampleConfig/base_apm_config.yaml index 7a11090e1c..e727fc6fb1 100644 --- a/translator/tocwconfig/sampleConfig/base_apm_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_apm_config.yaml @@ -1,6 +1,6 @@ connectors: {} exporters: - awsemf/apm: + awsemf/appsignals: certificate_file_path: "" detailed_metrics: false dimension_rollup_option: NoDimensionRollup @@ -10,7 +10,7 @@ exporters: enhanced_container_insights: false imds_retries: 0 local_mode: false - log_group_name: /aws/apm/generic + log_group_name: /aws/appsignals/generic log_retention: 0 log_stream_name: "" max_retries: 2 @@ -65,7 +65,7 @@ exporters: - Fault - Error metric_descriptors: [] - namespace: AWS/APM + namespace: AppSignals no_verify_ssl: false num_workers: 8 output_destination: cloudwatch @@ -81,7 +81,7 @@ exporters: role_arn: "" shared_credentials_file: [] version: "1" - awsxray/apm: + awsxray/appsignals: aws_log_groups: [] certificate_file_path: "" endpoint: https://fake_endpoint @@ -112,7 +112,7 @@ exporters: enabled: true include_metadata: true extensions: - awsproxy/apm: + awsproxy/appsignals: aws_endpoint: "" endpoint: 0.0.0.0:2000 local_mode: false @@ -120,7 +120,7 @@ extensions: region: "" role_arn: "" processors: - awsapm: + awsappsignals: resolvers: - generic rules: [] @@ -385,7 +385,7 @@ processors: timeout: 2s write_buffer_size: 0 receivers: - otlp/apm: + otlp/appsignals: protocols: grpc: auth: null @@ -411,24 +411,24 @@ receivers: traces_url_path: /v1/traces service: extensions: - - awsproxy/apm + - awsproxy/appsignals pipelines: - metrics/apm: + metrics/appsignals: exporters: - - awsemf/apm + - awsemf/appsignals processors: - resourcedetection - - awsapm + - awsappsignals receivers: - - otlp/apm - traces/apm: + - otlp/appsignals + traces/appsignals: exporters: - - awsxray/apm + - awsxray/appsignals processors: - resourcedetection - - awsapm + - awsappsignals receivers: - - otlp/apm + - otlp/appsignals telemetry: logs: development: false diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 96dd8916fa..2554f07b57 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -67,25 +67,25 @@ func TestBaseContainerInsightsConfig(t *testing.T) { checkTranslation(t, "base_container_insights_config", "linux", expectedEnvVars, "") } -func TestGenericAPMConfig(t *testing.T) { +func TestGenericAppSignalsConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(true) t.Setenv(config.HOST_NAME, "host_name_from_env") t.Setenv(config.HOST_IP, "127.0.0.1") expectedEnvVars := map[string]string{} - checkTranslation(t, "base_apm_config", "linux", expectedEnvVars, "") - checkTranslation(t, "base_apm_config", "windows", expectedEnvVars, "") + checkTranslation(t, "base_appsignals_config", "linux", expectedEnvVars, "") + checkTranslation(t, "base_appsignals_config", "windows", expectedEnvVars, "") } -func TestAPMAndKubernetesConfig(t *testing.T) { +func TestAppSignalsAndKubernetesConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(true) t.Setenv(config.HOST_NAME, "host_name_from_env") t.Setenv(config.HOST_IP, "127.0.0.1") - t.Setenv(common.KubernetesEnvVar, "use_apm_eks_config") + t.Setenv(common.KubernetesEnvVar, "use_appsignals_eks_config") expectedEnvVars := map[string]string{} - checkTranslation(t, "apm_and_kubernetes_config", "linux", expectedEnvVars, "") - checkTranslation(t, "apm_and_kubernetes_config", "windows", expectedEnvVars, "") + checkTranslation(t, "appsignals_and_kubernetes_config", "linux", expectedEnvVars, "") + checkTranslation(t, "appsignals_and_kubernetes_config", "windows", expectedEnvVars, "") } func TestEmfAndKubernetesConfig(t *testing.T) { diff --git a/translator/translate/otel/common/apm.go b/translator/translate/otel/common/apm.go index 5c6b4ab325..c6887bcf39 100644 --- a/translator/translate/otel/common/apm.go +++ b/translator/translate/otel/common/apm.go @@ -7,7 +7,7 @@ import "os" const KubernetesEnvVar = "K8S_NAMESPACE" -func IsAPMKubernetes() bool { +func IsAppSignalsKubernetes() bool { _, isSet := os.LookupEnv(KubernetesEnvVar) return isSet } diff --git a/translator/translate/otel/common/apm_test.go b/translator/translate/otel/common/apm_test.go index 2e04f5da79..5a4b8a0ba3 100644 --- a/translator/translate/otel/common/apm_test.go +++ b/translator/translate/otel/common/apm_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" ) -func TestIsAPMKubernetes(t *testing.T) { - assert.False(t, IsAPMKubernetes()) +func TestIsAppSignalsKubernetes(t *testing.T) { + assert.False(t, IsAppSignalsKubernetes()) t.Setenv(KubernetesEnvVar, "TEST") - assert.True(t, IsAPMKubernetes()) + assert.True(t, IsAppSignalsKubernetes()) } diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 2de6bb471b..6221491720 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -62,18 +62,18 @@ const ( PipelineNameHost = "host" PipelineNameHostDeltaMetrics = "hostDeltaMetrics" PipelineNameEmfLogs = "emf_logs" - APM = "apm" + AppSignals = "app_signals" - APMRules = "rules" + AppSignalsRules = "rules" ) var ( - APMTraces = ConfigKey(TracesKey, TracesCollectedKey, APM) - APMMetrics = ConfigKey(LogsKey, MetricsCollectedKey, APM) + AppSignalsTraces = ConfigKey(TracesKey, TracesCollectedKey, AppSignals) + AppSignalsMetrics = ConfigKey(LogsKey, MetricsCollectedKey, AppSignals) - APMConfigKeys = map[component.DataType]string{ - component.DataTypeTraces: APMTraces, - component.DataTypeMetrics: APMMetrics, + AppSignalsConfigKeys = map[component.DataType]string{ + component.DataTypeTraces: AppSignalsTraces, + component.DataTypeMetrics: AppSignalsMetrics, } ) diff --git a/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml similarity index 96% rename from translator/translate/otel/exporter/awsemf/apm_config_eks.yaml rename to translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml index 8c87e3bd55..19d72af94d 100644 --- a/translator/translate/otel/exporter/awsemf/apm_config_eks.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml @@ -1,5 +1,5 @@ -log_group_name: "/aws/apm/eks" -namespace: "AWS/APM" +log_group_name: "/aws/appsignals/eks" +namespace: "AppSignals" dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: diff --git a/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml similarity index 93% rename from translator/translate/otel/exporter/awsemf/apm_config_generic.yaml rename to translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml index c95e805a09..bba205d0ac 100644 --- a/translator/translate/otel/exporter/awsemf/apm_config_generic.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml @@ -1,5 +1,5 @@ -log_group_name: "/aws/apm/generic" -namespace: "AWS/APM" +log_group_name: "/aws/appsignals/generic" +namespace: "AppSignals" dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index b87ca44d60..632682ebfa 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -30,11 +30,11 @@ var defaultKubernetesConfig string //go:embed awsemf_default_prometheus.yaml var defaultPrometheusConfig string -//go:embed apm_config_eks.yaml -var apmConfigEks string +//go:embed appsignals_config_eks.yaml +var appSignalsConfigEks string -//go:embed apm_config_generic.yaml -var apmConfigGeneric string +//go:embed appsignals_config_generic.yaml +var appSignalsConfigGeneric string var ( ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) @@ -67,10 +67,10 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*awsemfexporter.Config) - if common.IsAPMKubernetes() && t.name == common.APM { - return common.GetYamlFileToYamlConfig(cfg, apmConfigEks) - } else if t.name == common.APM { - return common.GetYamlFileToYamlConfig(cfg, apmConfigGeneric) + if common.IsAppSignalsKubernetes() && t.name == common.AppSignals { + return common.GetYamlFileToYamlConfig(cfg, appSignalsConfigEks) + } else if t.name == common.AppSignals { + return common.GetYamlFileToYamlConfig(cfg, appSignalsConfigGeneric) } var defaultConfig string diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index a0d264647f..90cc9ce11b 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -690,32 +690,32 @@ func TestTranslator(t *testing.T) { } } -func TestTranslateAPM(t *testing.T) { - tt := NewTranslatorWithName(common.APM) +func TestTranslateAppSignals(t *testing.T) { + tt := NewTranslatorWithName(common.AppSignals) testCases := map[string]struct { input map[string]interface{} want *confmap.Conf wantErr error isKubernetes bool }{ - "WithAPMEnabledEKS": { + "WithAppSignalsEnabledEKS": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }}, - want: testutil.GetConf(t, filepath.Join("apm_config_eks.yaml")), + want: testutil.GetConf(t, filepath.Join("appsignals_config_eks.yaml")), isKubernetes: true, }, - "WithAPMEnabledGeneric": { + "WithAppSignalsEnabledGeneric": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }}, - want: testutil.GetConf(t, filepath.Join("apm_config_generic.yaml")), + want: testutil.GetConf(t, filepath.Join("appsignals_config_generic.yaml")), isKubernetes: false, }, } diff --git a/translator/translate/otel/exporter/awsxray/translator.go b/translator/translate/otel/exporter/awsxray/translator.go index 9b11902630..e2ce37477f 100644 --- a/translator/translate/otel/exporter/awsxray/translator.go +++ b/translator/translate/otel/exporter/awsxray/translator.go @@ -50,7 +50,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } cfg := t.factory.CreateDefaultConfig().(*awsxrayexporter.Config) - if isAPM(conf) { + if isAppSignals(conf) { cfg.IndexedAttributes = []string{ "aws.local.service", "aws.local.operation", "aws.remote.service", "aws.remote.operation", "HostedIn.EKS.Cluster", "HostedIn.K8s.Namespace", "K8s.RemoteNamespace", "aws.remote.target", @@ -115,6 +115,6 @@ func getRegion(conf *confmap.Conf) string { return region } -func isAPM(conf *confmap.Conf) bool { - return conf.IsSet(common.APMTraces) +func isAppSignals(conf *confmap.Conf) bool { + return conf.IsSet(common.AppSignalsTraces) } diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index 35973acff0..e381c58f42 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -51,11 +51,11 @@ func TestTranslator(t *testing.T) { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), }, - "WithAPMEnabled": { + "WithAppSignalsEnabled": { input: map[string]interface{}{ "traces": map[string]interface{}{ "traces_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }}, want: confmap.NewFromStringMap(map[string]interface{}{ diff --git a/translator/translate/otel/pipeline/apm/translator.go b/translator/translate/otel/pipeline/appsignals/translator.go similarity index 85% rename from translator/translate/otel/pipeline/apm/translator.go rename to translator/translate/otel/pipeline/appsignals/translator.go index cc6a40209d..15928c857f 100644 --- a/translator/translate/otel/pipeline/apm/translator.go +++ b/translator/translate/otel/pipeline/appsignals/translator.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package apm +package appsignals import ( "fmt" @@ -13,13 +13,13 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsxray" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/awsproxy" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/awsapm" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/awsappsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" ) const ( - pipelineName = "apm" + pipelineName = "app_signals" ) type translator struct { @@ -39,7 +39,7 @@ func (t *translator) ID() component.ID { } func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) { - configKey, ok := common.APMConfigKeys[t.dataType] + configKey, ok := common.AppSignalsConfigKeys[t.dataType] if !ok { return nil, fmt.Errorf("no config key defined for data type: %s", t.dataType) } @@ -48,17 +48,17 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } translators := &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap(otlp.NewTranslatorWithName(common.APM, otlp.WithDataType(t.dataType))), - Processors: common.NewTranslatorMap(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType)), awsapm.NewTranslator(awsapm.WithDataType(t.dataType))), + Receivers: common.NewTranslatorMap(otlp.NewTranslatorWithName(common.AppSignals, otlp.WithDataType(t.dataType))), + Processors: common.NewTranslatorMap(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType)), awsappsignals.NewTranslator(awsappsignals.WithDataType(t.dataType))), Exporters: common.NewTranslatorMap[component.Config](), Extensions: common.NewTranslatorMap[component.Config](), } if t.dataType == component.DataTypeTraces { - translators.Exporters.Set(awsxray.NewTranslatorWithName(common.APM)) - translators.Extensions.Set(awsproxy.NewTranslatorWithName(common.APM)) + translators.Exporters.Set(awsxray.NewTranslatorWithName(common.AppSignals)) + translators.Extensions.Set(awsproxy.NewTranslatorWithName(common.AppSignals)) } else { - translators.Exporters.Set(awsemf.NewTranslatorWithName(common.APM)) + translators.Exporters.Set(awsemf.NewTranslatorWithName(common.AppSignals)) } return translators, nil } diff --git a/translator/translate/otel/pipeline/apm/translator_test.go b/translator/translate/otel/pipeline/appsignals/translator_test.go similarity index 79% rename from translator/translate/otel/pipeline/apm/translator_test.go rename to translator/translate/otel/pipeline/appsignals/translator_test.go index f7ba4e5c28..3d60640714 100644 --- a/translator/translate/otel/pipeline/apm/translator_test.go +++ b/translator/translate/otel/pipeline/appsignals/translator_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package apm +package appsignals import ( "fmt" @@ -24,7 +24,7 @@ func TestTranslatorTraces(t *testing.T) { extensions []string } tt := NewTranslator(component.DataTypeTraces) - assert.EqualValues(t, "traces/apm", tt.ID().String()) + assert.EqualValues(t, "traces/appsignals", tt.ID().String()) testCases := map[string]struct { input map[string]interface{} want *want @@ -32,21 +32,21 @@ func TestTranslatorTraces(t *testing.T) { }{ "WithoutTracesCollectedKey": { input: map[string]interface{}{}, - wantErr: &common.MissingKeyError{ID: tt.ID(), JsonKey: fmt.Sprint(common.APMTraces)}, + wantErr: &common.MissingKeyError{ID: tt.ID(), JsonKey: fmt.Sprint(common.AppSignalsTraces)}, }, - "WithAPMEnabledTraces": { + "WithAppSignalsEnabledTraces": { input: map[string]interface{}{ "traces": map[string]interface{}{ "traces_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }, }, want: &want{ - receivers: []string{"otlp/apm"}, - processors: []string{"resourcedetection", "awsapm"}, - exporters: []string{"awsxray/apm"}, - extensions: []string{"awsproxy/apm"}, + receivers: []string{"otlp/appsignals"}, + processors: []string{"resourcedetection", "awsappsignals"}, + exporters: []string{"awsxray/appsignals"}, + extensions: []string{"awsproxy/appsignals"}, }, }, } @@ -74,7 +74,7 @@ func TestTranslatorMetrics(t *testing.T) { exporters []string } tt := NewTranslator(component.DataTypeMetrics) - assert.EqualValues(t, "metrics/apm", tt.ID().String()) + assert.EqualValues(t, "metrics/appsignals", tt.ID().String()) testCases := map[string]struct { input map[string]interface{} want *want @@ -82,20 +82,20 @@ func TestTranslatorMetrics(t *testing.T) { }{ "WithoutMetricsCollectedKey": { input: map[string]interface{}{}, - wantErr: &common.MissingKeyError{ID: tt.ID(), JsonKey: fmt.Sprint(common.APMMetrics)}, + wantErr: &common.MissingKeyError{ID: tt.ID(), JsonKey: fmt.Sprint(common.AppSignalsMetrics)}, }, - "WithAPMEnabledMetrics": { + "WithAppSignalsEnabledMetrics": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }, }, want: &want{ - receivers: []string{"otlp/apm"}, - processors: []string{"resourcedetection", "awsapm"}, - exporters: []string{"awsemf/apm"}, + receivers: []string{"otlp/appsignals"}, + processors: []string{"resourcedetection", "awsappsignals"}, + exporters: []string{"awsemf/appsignals"}, }, }, } diff --git a/translator/translate/otel/processor/awsapm/testdata/config_eks.yaml b/translator/translate/otel/processor/awsappsignals/testdata/config_eks.yaml similarity index 100% rename from translator/translate/otel/processor/awsapm/testdata/config_eks.yaml rename to translator/translate/otel/processor/awsappsignals/testdata/config_eks.yaml diff --git a/translator/translate/otel/processor/awsapm/testdata/config_generic.yaml b/translator/translate/otel/processor/awsappsignals/testdata/config_generic.yaml similarity index 100% rename from translator/translate/otel/processor/awsapm/testdata/config_generic.yaml rename to translator/translate/otel/processor/awsappsignals/testdata/config_generic.yaml diff --git a/translator/translate/otel/processor/awsapm/testdata/invalidRulesConfig.json b/translator/translate/otel/processor/awsappsignals/testdata/invalidRulesConfig.json similarity index 93% rename from translator/translate/otel/processor/awsapm/testdata/invalidRulesConfig.json rename to translator/translate/otel/processor/awsappsignals/testdata/invalidRulesConfig.json index 17f9fc537d..1c8bd7e378 100644 --- a/translator/translate/otel/processor/awsapm/testdata/invalidRulesConfig.json +++ b/translator/translate/otel/processor/awsappsignals/testdata/invalidRulesConfig.json @@ -1,7 +1,7 @@ { "logs": { "metrics_collected": { - "apm": { + "app_signals": { "rules": [ { "selectors": [ diff --git a/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.json b/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfig.json similarity index 98% rename from translator/translate/otel/processor/awsapm/testdata/validRulesConfig.json rename to translator/translate/otel/processor/awsappsignals/testdata/validRulesConfig.json index bf3c011720..7d6252542d 100644 --- a/translator/translate/otel/processor/awsapm/testdata/validRulesConfig.json +++ b/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfig.json @@ -1,7 +1,7 @@ { "logs": { "metrics_collected": { - "apm": { + "app_signals": { "rules": [ { "selectors": [ diff --git a/translator/translate/otel/processor/awsapm/testdata/validRulesConfigEKS.yaml b/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigEKS.yaml similarity index 100% rename from translator/translate/otel/processor/awsapm/testdata/validRulesConfigEKS.yaml rename to translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigEKS.yaml diff --git a/translator/translate/otel/processor/awsapm/testdata/validRulesConfigGeneric.yaml b/translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigGeneric.yaml similarity index 100% rename from translator/translate/otel/processor/awsapm/testdata/validRulesConfigGeneric.yaml rename to translator/translate/otel/processor/awsappsignals/testdata/validRulesConfigGeneric.yaml diff --git a/translator/translate/otel/processor/awsapm/translator.go b/translator/translate/otel/processor/awsappsignals/translator.go similarity index 85% rename from translator/translate/otel/processor/awsapm/translator.go rename to translator/translate/otel/processor/awsappsignals/translator.go index 6126778cba..fe197250a1 100644 --- a/translator/translate/otel/processor/awsapm/translator.go +++ b/translator/translate/otel/processor/awsappsignals/translator.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsapm +package awsappsignals import ( _ "embed" @@ -11,8 +11,8 @@ import ( "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/processor" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/customconfiguration" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) @@ -43,7 +43,7 @@ func WithDataType(dataType component.DataType) Option { var _ common.Translator[component.Config] = (*translator)(nil) func NewTranslator(opts ...Option) common.Translator[component.Config] { - t := &translator{factory: awsapmprocessor.NewFactory()} + t := &translator{factory: awsappsignals.NewFactory()} for _, opt := range opts { opt.apply(t) } @@ -55,9 +55,9 @@ func (t *translator) ID() component.ID { } func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - configKey := common.APMConfigKeys[t.dataType] - cfg := t.factory.CreateDefaultConfig().(*awsapmprocessor.Config) - if common.IsAPMKubernetes() { + configKey := common.AppSignalsConfigKeys[t.dataType] + cfg := t.factory.CreateDefaultConfig().(*awsappsignals.Config) + if common.IsAppSignalsKubernetes() { cfg.Resolvers = []string{"eks"} } else { cfg.Resolvers = []string{"generic"} @@ -65,9 +65,9 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { return t.translateCustomRules(conf, configKey, cfg) } -func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, cfg *awsapmprocessor.Config) (component.Config, error) { +func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, cfg *awsappsignals.Config) (component.Config, error) { var rules []customconfiguration.Rule - rulesConfigKey := common.ConfigKey(configKey, common.APMRules) + rulesConfigKey := common.ConfigKey(configKey, common.AppSignalsRules) if conf.IsSet(rulesConfigKey) { for _, rule := range conf.Get(rulesConfigKey).([]interface{}) { ruleConfig := customconfiguration.Rule{} diff --git a/translator/translate/otel/processor/awsapm/translator_test.go b/translator/translate/otel/processor/awsappsignals/translator_test.go similarity index 59% rename from translator/translate/otel/processor/awsapm/translator_test.go rename to translator/translate/otel/processor/awsappsignals/translator_test.go index 82cf6b2985..217563e274 100644 --- a/translator/translate/otel/processor/awsapm/translator_test.go +++ b/translator/translate/otel/processor/awsappsignals/translator_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package awsapm +package awsappsignals import ( _ "embed" @@ -14,29 +14,29 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" - "github.com/aws/amazon-cloudwatch-agent/processor/awsapmprocessor" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) var ( //go:embed testdata/config_eks.yaml - validAPMYamlEKS string + validAppSignalsYamlEKS string //go:embed testdata/config_generic.yaml - validAPMYamlGeneric string + validAppSignalsYamlGeneric string //go:embed testdata/validRulesConfig.json - validAPMRulesConfig string + validAppSignalsRulesConfig string //go:embed testdata/validRulesConfigEKS.yaml - validAPMRulesYamlEKS string + validAppSignalsRulesYamlEKS string //go:embed testdata/validRulesConfigGeneric.yaml - validAPMRulesYamlGeneric string + validAppSignalsRulesYamlGeneric string //go:embed testdata/invalidRulesConfig.json - invalidAPMRulesConfig string + invalidAppSignalsRulesConfig string ) func TestTranslate(t *testing.T) { var validJsonMap, invalidJsonMap map[string]interface{} - json.Unmarshal([]byte(validAPMRulesConfig), &validJsonMap) - json.Unmarshal([]byte(invalidAPMRulesConfig), &invalidJsonMap) + json.Unmarshal([]byte(validAppSignalsRulesConfig), &validJsonMap) + json.Unmarshal([]byte(invalidAppSignalsRulesConfig), &invalidJsonMap) tt := NewTranslator(WithDataType(component.DataTypeMetrics)) testCases := map[string]struct { @@ -45,44 +45,44 @@ func TestTranslate(t *testing.T) { wantErr error isKubernetes bool }{ - //The config for the awsapm processor is https://code.amazon.com/packages/AWSTracingSamplePetClinic/blobs/97ce3c409986ac8ae014de1e3fe71fdb98080f22/--/eks/apm/auto-instrumentation-new.yaml#L20 - //The awsapm processor config does not have a platform field, instead it gets added to resolvers when marshalled - "WithAPMEnabledEKS": { + //The config for the awsappsignals processor is https://code.amazon.com/packages/AWSTracingSamplePetClinic/blobs/97ce3c409986ac8ae014de1e3fe71fdb98080f22/--/eks/appsignals/auto-instrumentation-new.yaml#L20 + //The awsappsignals processor config does not have a platform field, instead it gets added to resolvers when marshalled + "WithAppSignalsEnabledEKS": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }}, - want: validAPMYamlEKS, + want: validAppSignalsYamlEKS, isKubernetes: true, }, - "WithAPMCustomRulesEnabledEKS": { + "WithAppSignalsCustomRulesEnabledEKS": { input: validJsonMap, - want: validAPMRulesYamlEKS, + want: validAppSignalsRulesYamlEKS, isKubernetes: true, }, - "WithAPMEnabledGeneric": { + "WithAppSignalsEnabledGeneric": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }}, - want: validAPMYamlGeneric, + want: validAppSignalsYamlGeneric, isKubernetes: false, }, - "WithAPMCustomRulesEnabledGeneric": { + "WithAppSignalsCustomRulesEnabledGeneric": { input: validJsonMap, - want: validAPMRulesYamlGeneric, + want: validAppSignalsRulesYamlGeneric, isKubernetes: false, }, - "WithInvalidAPMCustomRulesEnabled": { + "WithInvalidAppSignalsCustomRulesEnabled": { input: invalidJsonMap, wantErr: errors.New("replace action set, but no replacements defined for service rule"), }, } - factory := awsapmprocessor.NewFactory() + factory := awsappsignals.NewFactory() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { if testCase.isKubernetes { @@ -93,12 +93,12 @@ func TestTranslate(t *testing.T) { assert.Equal(t, testCase.wantErr, err) if err == nil { require.NotNil(t, got) - gotCfg, ok := got.(*awsapmprocessor.Config) + gotCfg, ok := got.(*awsappsignals.Config) require.True(t, ok) wantCfg := factory.CreateDefaultConfig() yamlConfig, err := common.GetYamlFileToYamlConfig(wantCfg, testCase.want) require.NoError(t, err) - assert.Equal(t, yamlConfig.(*awsapmprocessor.Config), gotCfg) + assert.Equal(t, yamlConfig.(*awsappsignals.Config), gotCfg) } }) } diff --git a/translator/translate/otel/processor/resourcedetection/translator_test.go b/translator/translate/otel/processor/resourcedetection/translator_test.go index f022ce2f2e..d84624937f 100644 --- a/translator/translate/otel/processor/resourcedetection/translator_test.go +++ b/translator/translate/otel/processor/resourcedetection/translator_test.go @@ -20,11 +20,11 @@ func TestTranslate(t *testing.T) { want *confmap.Conf wantErr error }{ - "WithAPMEnabled": { + "WithAppSignalsEnabled": { input: map[string]interface{}{ "traces": map[string]interface{}{ "traces_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }}, want: confmap.NewFromStringMap(map[string]interface{}{ diff --git a/translator/translate/otel/receiver/otlp/apm_config.yaml b/translator/translate/otel/receiver/otlp/appsignals_config.yaml similarity index 100% rename from translator/translate/otel/receiver/otlp/apm_config.yaml rename to translator/translate/otel/receiver/otlp/appsignals_config.yaml diff --git a/translator/translate/otel/receiver/otlp/translator.go b/translator/translate/otel/receiver/otlp/translator.go index 4573d017b8..48c86c0fd3 100644 --- a/translator/translate/otel/receiver/otlp/translator.go +++ b/translator/translate/otel/receiver/otlp/translator.go @@ -25,8 +25,8 @@ var ( component.DataTypeTraces: common.ConfigKey(common.TracesKey, common.TracesCollectedKey, common.OtlpKey), } - //go:embed apm_config.yaml - apmConfig string + //go:embed appsignals_config.yaml + appSignalsConfig string ) type translator struct { @@ -79,8 +79,8 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*otlpreceiver.Config) // TODO: Should follow pattern done in awsemf and awsexray exporter translations (i.e should be integrated with standard otlp translation) - if t.name == common.APM { - return common.GetYamlFileToYamlConfig(cfg, apmConfig) + if t.name == common.AppSignals { + return common.GetYamlFileToYamlConfig(cfg, appSignalsConfig) } configKey, ok := configKeys[t.dataType] diff --git a/translator/translate/otel/receiver/otlp/translator_test.go b/translator/translate/otel/receiver/otlp/translator_test.go index c7ba2385d8..06a6672a1e 100644 --- a/translator/translate/otel/receiver/otlp/translator_test.go +++ b/translator/translate/otel/receiver/otlp/translator_test.go @@ -75,18 +75,18 @@ func TestTracesTranslator(t *testing.T) { } } -func TestTranslateAPM(t *testing.T) { - tt := NewTranslatorWithName(common.APM, WithDataType(component.DataTypeTraces)) +func TestTranslateAppSignals(t *testing.T) { + tt := NewTranslatorWithName(common.AppSignals, WithDataType(component.DataTypeTraces)) testCases := map[string]struct { input map[string]interface{} want *confmap.Conf wantErr error }{ - "WithAPMEnabledTraces": { + "WithAppSignalsEnabledTraces": { input: map[string]interface{}{ "traces": map[string]interface{}{ "traces_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }}, want: confmap.NewFromStringMap(map[string]interface{}{ diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 4dd8a02f9a..9e53cf1689 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -21,7 +21,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/apm" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/appsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsights" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/emf_logs" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/host" @@ -67,8 +67,8 @@ func Translate(jsonConfig interface{}, os string) (*otelcol.Config, error) { }) translators := common.NewTranslatorMap( - apm.NewTranslator(component.DataTypeTraces), - apm.NewTranslator(component.DataTypeMetrics), + appsignals.NewTranslator(component.DataTypeTraces), + appsignals.NewTranslator(component.DataTypeMetrics), host.NewTranslator(common.PipelineNameHost, hostReceivers), host.NewTranslator(common.PipelineNameHostDeltaMetrics, deltaMetricsReceivers), containerinsights.NewTranslator(), diff --git a/translator/translate/otel/translate_otel_test.go b/translator/translate/otel/translate_otel_test.go index d0e48e4146..f63df7e725 100644 --- a/translator/translate/otel/translate_otel_test.go +++ b/translator/translate/otel/translate_otel_test.go @@ -46,34 +46,34 @@ func TestTranslator(t *testing.T) { }, }, }, - "WithAPMMetricsEnabled": { + "WithAppSignalsMetricsEnabled": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }, }, }, - "WithAPMTracesEnabled": { + "WithAppSignalsTracesEnabled": { input: map[string]interface{}{ "traces": map[string]interface{}{ "traces_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }, }, }, - "WithAPMMetricsAndTracesEnabled": { + "WithAppSignalsMetricsAndTracesEnabled": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }, "traces": map[string]interface{}{ "traces_collected": map[string]interface{}{ - "apm": map[string]interface{}{}, + "app_signals": map[string]interface{}{}, }, }, }, From 677e9e11f341eb629a37818383e3ac11996d2380 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Wed, 25 Oct 2023 17:16:23 -0400 Subject: [PATCH 22/38] cleanup --- .github/repo_sync_pr_template.md | 46 ---- .github/workflows/PR-build.yml | 144 ----------- .github/workflows/apm-beta-pre-release.yml | 31 --- .github/workflows/apm-e2e-test.yml | 244 ------------------ .github/workflows/repo-sync.yml | 58 ----- service/defaultcomponents/components.go | 2 +- service/defaultcomponents/components_test.go | 3 + translator/tocwconfig/tocwconfig_test.go | 2 + .../otel/common/{apm.go => appsignals.go} | 0 .../{apm_test.go => appsignals_test.go} | 0 translator/translate/otel/common/common.go | 5 +- .../otel/exporter/awsemf/translator_test.go | 7 +- 12 files changed, 14 insertions(+), 528 deletions(-) delete mode 100644 .github/repo_sync_pr_template.md delete mode 100644 .github/workflows/PR-build.yml delete mode 100644 .github/workflows/apm-beta-pre-release.yml delete mode 100644 .github/workflows/apm-e2e-test.yml delete mode 100644 .github/workflows/repo-sync.yml rename translator/translate/otel/common/{apm.go => appsignals.go} (100%) rename translator/translate/otel/common/{apm_test.go => appsignals_test.go} (100%) diff --git a/.github/repo_sync_pr_template.md b/.github/repo_sync_pr_template.md deleted file mode 100644 index 8d610babbc..0000000000 --- a/.github/repo_sync_pr_template.md +++ /dev/null @@ -1,46 +0,0 @@ -# Description of the issue -An automated PR to kickstart the process of syncing the latest changes from [cw-agent](https://github.com/aws/amazon-cloudwatch-agent/) - -# Description of changes - -### Follow the git CLI instructions resolve the merge conflicts - -```shell -git pull origin main -git checkout repo-sync-- -git merge main # do a regular merge -- we want to keep the commits -# resolve merge conflicts in your preferred IDE -git push -u origin repo-sync-- -``` - -Some useful commands -* [Restore conflict resolution in a single file](https://stackoverflow.com/questions/14409420/restart-undo-conflict-resolution-in-a-single-file) - `git checkout -m ` -* Total reset - `git merge --abort` - -### Related docs -* Resolving conflicts with: - * [Git CLI](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-using-the-command-line) - * [IntelliJ](https://www.jetbrains.com/help/idea/resolving-conflicts.html#distributed-version-control-systems) - * [GoLand](https://www.jetbrains.com/help/go/resolve-conflicts.html) - * [VSCode](https://learn.microsoft.com/en-us/visualstudio/version-control/git-resolve-conflicts?view=vs-2022) - -### Best practices - -* Remember to update all references from `amazon-cloudwatch-agent` to `private-amazon-cloudwatch-agent-staging` -* Resolve the `go.sum` with `go mod tidy`. Don't bother manually resolving conflicts in this file -* When finished, ensure builds work by using `make build` or `make release` -* When unsure or blocked, do a deep dive on the `git blame` for greater context. Maybe even look for the associated PR's and ask the original authors and PR approvers -* If another automated PR arrives before your work is merged, just close your current one and save the branch -* After your PR is approved, **do a regular merge to preserve the commits**. -* Remember to cleanup your commits because none of them will be squashed in a regular merge - -# License -By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. - -# Tests -n/a - -# Requirements -_Before commit the code, please do the following steps._ -1. Run `make fmt` and `make fmt-sh` -2. Run `make lint` \ No newline at end of file diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml deleted file mode 100644 index 9bada91b1d..0000000000 --- a/.github/workflows/PR-build.yml +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: MIT - -name: PR Build -on: - workflow_dispatch: - pull_request: - branches: - - main* - types: - - opened - - synchronize - - reopened - - ready_for_review - -concurrency: - group: ${{ github.workflow }}-${{ github.ref_name }} - cancel-in-progress: true - -jobs: - changes: - name: Check changes - runs-on: ubuntu-latest - outputs: - build: ${{ steps.filter.outputs.build }} - lint: ${{ steps.filter.outputs.lint }} - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - list-files: shell - filters: .github/config/file-filters.yml - - - name: List all updated files - run: | - for file in ${{ steps.filter.outputs.build_files }}; do - echo "$file" - done - - lint: - needs: [changes] - name: Check lint - runs-on: ubuntu-latest - steps: - - name: Set up Go 1.x - if: needs.changes.outputs.lint == 'true' - uses: actions/setup-go@v4 - with: - go-version: ~1.21.1 - cache: false - - - name: Check out code - if: needs.changes.outputs.lint == 'true' - uses: actions/checkout@v3 - - - name: Check format - if: needs.changes.outputs.lint == 'true' - run: | - make fmt fmt-sh - if [ ! -z "`git status --porcelain`" ]; then - echo "make fmt changed files" - git status - exit 1 - fi - - - name: Check license and imports - if: needs.changes.outputs.lint == 'true' - run: make simple-lint - - build: - needs: [lint, changes] - name: Build ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest, windows-2019, windows-latest, macos-11] - include: - - os: ubuntu-latest - family: linux - cache-path: | - ~/.cache/go-build - ~/go/pkg/mod - - os: macos-11 - family: darwin - cache-path: | - ~/Library/Caches/go-build - ~/go/pkg/mod - - os: windows-2019 - family: windows - cache-path: | - ~\AppData\Local\go-build - ~\go\pkg\mod - - os: windows-latest - family: windows - cache-path: | - ~\AppData\Local\go-build - ~\go\pkg\mod - steps: - - name: Set up Go 1.x - if: needs.changes.outputs.build == 'true' - uses: actions/setup-go@v4 - with: - go-version: ~1.21.1 - cache: false - - - name: Check out code - if: needs.changes.outputs.build == 'true' - uses: actions/checkout@v3 - - - name: Cache binaries - id: cached_binaries - if: needs.changes.outputs.build == 'true' - uses: actions/cache@v3 - with: - key: "cached-binaries-${{ matrix.os }}-${{ github.sha }}" - path: go.mod - - - name: Cache build output - if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - uses: actions/cache@v3 - with: - path: ${{ matrix.cache-path }} - key: v1-go-pkg-mod-${{ matrix.os }}-${{ hashFiles('**/go.sum') }} - - - name: Install make - if: matrix.family == 'windows' && steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - run: choco install make - - - name: Unit Test - if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - run: make test - - - name: Upload coverage to Codecov - if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - uses: codecov/codecov-action@v3 - with: - verbose: true - - - name: Build - if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' - run: make amazon-cloudwatch-agent-${{ matrix.family }} - \ No newline at end of file diff --git a/.github/workflows/apm-beta-pre-release.yml b/.github/workflows/apm-beta-pre-release.yml deleted file mode 100644 index ca201d50b6..0000000000 --- a/.github/workflows/apm-beta-pre-release.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: MIT - -name: APM Beta Pre-Release -on: - push: - branches: - - main-apm -jobs: - BuildAndUpload: - uses: ./.github/workflows/test-build.yml - secrets: inherit - permissions: - id-token: write - contents: read - with: - ContainerRepositoryNameAndTag: "apm-beta-pre-release:latest" - BucketKey: "apm-beta-pre-release" - PackageBucketKey: "apm-beta-pre-release" - - e2e-test: - needs: BuildAndUpload - uses: ./.github/workflows/apm-e2e-test.yml - secrets: inherit - concurrency: - group: 'pulse-cw-agent-test' - cancel-in-progress: false - with: - test-cluster-name: 'pulse-cw-agent-test' - # The following needs to be updated once we go public - apm-cwagent-image-name: '506463145083.dkr.ecr.us-west-2.amazonaws.com/apm-beta-pre-release:latest' \ No newline at end of file diff --git a/.github/workflows/apm-e2e-test.yml b/.github/workflows/apm-e2e-test.yml deleted file mode 100644 index 52d89ee8b1..0000000000 --- a/.github/workflows/apm-e2e-test.yml +++ /dev/null @@ -1,244 +0,0 @@ -# This is a reusable workflow for running the E2E test for APM. -# It is meant to be called from another workflow. -# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview -name: APM Enablement E2E Testing -on: - workflow_call: - inputs: - test-cluster-name: - required: true - type: string - apm-cwagent-image-name: - required: true - type: string - -permissions: - id-token: write - contents: read - -env: - AWS_DEFAULT_REGION: us-east-1 - SAMPLE_APP_NAMESPACE: sample-app-namespace - METRIC_NAMESPACE: AWS/APM - APM_LOG_GROUP: /aws/apm/eks - -jobs: - apm-e2e-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - # Checkout apm branch and get only the required resources for testing - ref: apm - sparse-checkout: | - test - - - name: Generate testing id - run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.APM_E2E_TEST_ROLE_ARN }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - - # local directory to store the kubernetes config - - name: Create kubeconfig directory - run: mkdir -p ${{ github.workspace }}/.kube - - - name: Set KUBECONFIG environment variable - run: echo KUBECONFIG="${{ github.workspace }}/.kube/config" >> $GITHUB_ENV - - - name: Install eksctl - run: | - mkdir ${{ github.workspace }}/eksctl - curl -sLO "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_Linux_amd64.tar.gz" - tar -xzf eksctl_Linux_amd64.tar.gz -C ${{ github.workspace }}/eksctl && rm eksctl_Linux_amd64.tar.gz - echo "${{ github.workspace }}/eksctl" >> $GITHUB_PATH - - # Create the K8s service account to grant the sample app - # permissions to call AWS services from within the pods. - # https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html - # - # This needs to be done before launching the sample app - # because the sample app manifest requires the service - # account to be created upfront. - - name: Enable OIDC for cluster - run: eksctl utils associate-iam-oidc-provider --cluster ${{ inputs.test-cluster-name }} --region ${{ env.AWS_DEFAULT_REGION }} --approve - - - name: Create role for AWS access from the sample app - id: create_service_account - run: | - eksctl create iamserviceaccount \ - --name service-account-${{ env.TESTING_ID }} \ - --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ - --cluster ${{ inputs.test-cluster-name }} \ - --role-name eks-s3-access-${{ env.TESTING_ID }} \ - --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \ - --region ${{ env.AWS_DEFAULT_REGION }} \ - --approve - - - name: Set up terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_wrapper: false - - - name: Deploy sample app via terraform - working-directory: test/terraform/eks - run: | - terraform init - terraform validate - terraform apply -auto-approve \ - -var="test_id=${{ env.TESTING_ID }}" \ - -var="kube_directory_path=${{ github.workspace }}/.kube" \ - -var="eks_cluster_name=${{ inputs.test-cluster-name }}" \ - -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \ - -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \ - -var="sample_app_image=${{ secrets.APM_E2E_SAMPLE_APP_FRONTEND_SERVICE_IMAGE }}" \ - -var="sample_remote_app_image=${{ secrets.APM_E2E_SAMPLE_APP_REMOTE_SERVICE_IMAGE }}" - - # Enable APM on the test cluster - - name: Pull and unzip enablement script from S3 - run: aws s3 cp ${{ secrets.APM_E2E_ONBOARDING_ZIP_S3_URI }} . && unzip -j onboarding.zip - - - name: Change OTEL_TRACES_SAMPLER to always_on - run: "sed -i 's#parentbased_traceidratio#always_on#g' instrumentation.yaml" - - - name: Set the CW Agent image in the manifest - run: yq -i '.spec.image = "${{ inputs.apm-cwagent-image-name }}"' agent-daemon-set.yaml - - - name: Enable APM - run: | - ./enable-apm.sh \ - ${{ inputs.test-cluster-name }} \ - ${{ env. AWS_DEFAULT_REGION }} \ - ${{ env.SAMPLE_APP_NAMESPACE }} - - # Application pods need to be restarted for the - # apm instrumentation to take effect - - name: Restart the app pods - run: kubectl delete pods --all -n ${{ env.SAMPLE_APP_NAMESPACE }} - - - name: Wait for sample app pods to come up - run: | - kubectl wait --for=condition=Ready pod --all -n ${{ env.SAMPLE_APP_NAMESPACE }} \ - - - name: Verify pod ADOT image - run: | - kubectl get pods -n ${{ env.SAMPLE_APP_NAMESPACE }} --output json | \ - jq '.items[0].status.initContainerStatuses[0].imageID' - - - name: Verify pod CWAgent image - run: | - kubectl get pods -n amazon-cloudwatch --output json | \ - jq '.items[0].status.containerStatuses[0].imageID' - - - name: Get the sample app endpoint - run: | - echo "APP_ENDPOINT=$(terraform output sample_app_endpoint)" >> $GITHUB_ENV - working-directory: test/terraform/eks - - - name: Wait for app endpoint to come online - run: | - attempt_counter=0 - max_attempts=12 - until $(curl --output /dev/null --silent --head --fail http://${{ env.APP_ENDPOINT }}); do - if [ ${attempt_counter} -eq ${max_attempts} ];then - echo "Max attempts reached" - exit 1 - fi - - printf '.' - attempt_counter=$(($attempt_counter+1)) - sleep 10 - done - - # Validation for pulse telemetry data - - name: Call endpoint and validate generated EMF logs - id: log-validation - working-directory: test/validator - run: ./gradlew run --args='-c log-validation.yml - --testing-id ${{ env.TESTING_ID }} - --endpoint http://${{ env.APP_ENDPOINT }} - --region ${{ env.AWS_DEFAULT_REGION }} - --account-id ${{ secrets.APM_E2E_TEST_ACCOUNT }} - --metric-namespace ${{ env.METRIC_NAMESPACE }} - --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} - --cluster ${{ inputs.test-cluster-name }} - --service-name sample-application-${{ env.TESTING_ID }} - --rollup' - - - name: Call endpoints and validate generated metrics - id: metric-validation - if: success() || steps.log-validation.outcome == 'failure' - working-directory: test/validator - run: ./gradlew run --args='-c metric-validation.yml - --endpoint http://${{ env.APP_ENDPOINT }} - --region ${{ env.AWS_DEFAULT_REGION }} - --account-id ${{ secrets.APM_E2E_TEST_ACCOUNT }} - --metric-namespace ${{ env.METRIC_NAMESPACE }} - --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} - --cluster ${{ inputs.test-cluster-name }} - --service-name sample-application-${{ env.TESTING_ID }} - --remote-service-name sample-remote-application-${{ env.TESTING_ID }} - --rollup' - - - name: Call endpoints and validate generated traces - if: success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' - working-directory: test/validator - run: ./gradlew run --args='-c trace-validation.yml - --endpoint http://${{ env.APP_ENDPOINT }} - --region ${{ env.AWS_DEFAULT_REGION }} - --account-id ${{ secrets.APM_E2E_TEST_ACCOUNT }} - --metric-namespace ${{ env.METRIC_NAMESPACE }} - --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }} - --cluster ${{ inputs.test-cluster-name }} - --service-name sample-application-${{ env.TESTING_ID }} - --remote-service-name sample-remote-application-${{ env.TESTING_ID }} - --rollup' - - # Clean up Procedures - - - name: Remove log group deletion command - if: always() - run: | - delete_log_group="aws logs delete-log-group --log-group-name '${{ env.APM_LOG_GROUP }}' --region \$REGION" - sed -i "s#$delete_log_group##g" clean-apm.sh - - - name: Clean APM - if: always() - run: | - ./clean-apm.sh \ - ${{ inputs.test-cluster-name }} \ - ${{ env. AWS_DEFAULT_REGION }} \ - ${{ env.SAMPLE_APP_NAMESPACE }} - - # This step also deletes lingering resources from previous test runs - - name: Delete all sample app resources - if: always() - continue-on-error: true - timeout-minutes: 10 - run: kubectl delete namespace ${{ env.SAMPLE_APP_NAMESPACE }} - - - name: Terraform destroy - if: always() - continue-on-error: true - run: | - cd test/terraform/eks - terraform destroy -auto-approve \ - -var="test_id=${{ env.TESTING_ID }}" \ - -var="kube_directory_path=${{ github.workspace }}/.kube" \ - -var="eks_cluster_name=${{ inputs.test-cluster-name }}" \ - -var="test_namespace=${{ env.SAMPLE_APP_NAMESPACE }}" \ - -var="service_account_aws_access=service-account-${{ env.TESTING_ID }}" \ - -var="sample_app_image=${{ env.SAMPLE_APP_IMAGE }}" - - - name: Remove aws access service account - if: always() - continue-on-error: true - run: | - eksctl delete iamserviceaccount \ - --name service-account-${{ env.TESTING_ID }} \ - --namespace ${{ env.SAMPLE_APP_NAMESPACE }} \ - --cluster ${{ inputs.test-cluster-name }} \ - --region ${{ env.AWS_DEFAULT_REGION }} \ \ No newline at end of file diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml deleted file mode 100644 index 28bd6ea6bb..0000000000 --- a/.github/workflows/repo-sync.yml +++ /dev/null @@ -1,58 +0,0 @@ -# disable this workflow after beta phase -name: Sync with upstream - -on: - schedule: - - cron: "0 0 * * 0" # every sunday at 12AM - workflow_dispatch: - -env: - RUN_ID: "${{ github.run_number }}.${{ github.run_attempt }}" - UPSTREAM: "https://github.com/aws/amazon-cloudwatch-agent.git" - -jobs: - # gets the last commit hash from public/master and defines the branch name - # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idoutputs - define-branch-name: - runs-on: ubuntu-latest - steps: - - name: Get last commit hash from public - id: get-last-commit - run: echo "hash=$(git ls-remote ${{ env.UPSTREAM }} HEAD | awk '{print $1;}')" >> $GITHUB_OUTPUT - outputs: - LAST_COMMIT: ${{ steps.get-last-commit.outputs.hash }} - PR_BRANCH: "repo-sync-${{ steps.get-last-commit.outputs.hash }}-${{ env.RUN_ID }}" - - # pushes the latest from public/main to private/repo-sync - # https://github.com/marketplace/actions/github-repo-sync - create-branch: - needs: define-branch-name - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - name: repo-sync - uses: repo-sync/github-sync@v2 - with: - source_repo: ${{ env.UPSTREAM }} - source_branch: "main" - destination_branch: ${{ needs.define-branch-name.outputs.PR_BRANCH }} - github_token: ${{ secrets.WILLIAZZ_PAT }} - - # upon create-branch completion, creates a PR from private/repo-sync to private/main - # https://github.com/marketplace/actions/github-pull-request-action - create-pr: - needs: [define-branch-name, create-branch] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: pull-request - uses: repo-sync/pull-request@v2 - with: - source_branch: ${{ needs.define-branch-name.outputs.PR_BRANCH }} - destination_branch: "main-apm" - github_token: ${{ secrets.GITHUB_TOKEN }} - pr_title: "Automated sync with upstream - last commit ${{ needs.define-branch-name.outputs.LAST_COMMIT }} - run #${{ env.RUN_ID }}" - pr_template: ".github/repo_sync_pr_template.md" - pr_allow_empty: false \ No newline at end of file diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index b288b0deae..602371fa40 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -27,8 +27,8 @@ import ( "github.com/aws/amazon-cloudwatch-agent/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/plugins/outputs/cloudwatch" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/ec2tagger" ) func Factories() (otelcol.Factories, error) { diff --git a/service/defaultcomponents/components_test.go b/service/defaultcomponents/components_test.go index f21ae40ce5..17369775b4 100644 --- a/service/defaultcomponents/components_test.go +++ b/service/defaultcomponents/components_test.go @@ -29,6 +29,7 @@ func TestComponents(t *testing.T) { processors := factories.Processors assert.Len(t, processors, processorCount) + assert.NotNil(t, processors["awsappsignals"]) assert.NotNil(t, processors["batch"]) assert.NotNil(t, processors["cumulativetodelta"]) assert.NotNil(t, processors["ec2tagger"]) @@ -45,4 +46,6 @@ func TestComponents(t *testing.T) { extensions := factories.Extensions assert.Len(t, extensions, extensionsCount) + assert.NotNil(t, extensions["agenthealth"]) + assert.NotNil(t, extensions["awsproxy"]) } diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 4c7caa6dc8..106d45cd88 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -65,6 +65,7 @@ func TestBaseContainerInsightsConfig(t *testing.T) { "AWS_CA_BUNDLE": "/etc/test/ca_bundle.pem", } checkTranslation(t, "base_container_insights_config", "linux", expectedEnvVars, "") + checkTranslation(t, "base_container_insights_config", "darwin", nil, "") } func TestGenericAppSignalsConfig(t *testing.T) { @@ -96,6 +97,7 @@ func TestEmfAndKubernetesConfig(t *testing.T) { t.Setenv(config.HOST_IP, "127.0.0.1") expectedEnvVars := map[string]string{} checkTranslation(t, "emf_and_kubernetes_config", "linux", expectedEnvVars, "") + checkTranslation(t, "emf_and_kubernetes_config", "darwin", nil, "") } func TestKubernetesModeOnPremiseConfig(t *testing.T) { diff --git a/translator/translate/otel/common/apm.go b/translator/translate/otel/common/appsignals.go similarity index 100% rename from translator/translate/otel/common/apm.go rename to translator/translate/otel/common/appsignals.go diff --git a/translator/translate/otel/common/apm_test.go b/translator/translate/otel/common/appsignals_test.go similarity index 100% rename from translator/translate/otel/common/apm_test.go rename to translator/translate/otel/common/appsignals_test.go diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 6221491720..1e1bef0e73 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -62,9 +62,8 @@ const ( PipelineNameHost = "host" PipelineNameHostDeltaMetrics = "hostDeltaMetrics" PipelineNameEmfLogs = "emf_logs" - AppSignals = "app_signals" - - AppSignalsRules = "rules" + AppSignals = "app_signals" + AppSignalsRules = "rules" ) var ( diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index d7ce6559a1..0b05f10ddb 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -4,15 +4,20 @@ package awsemf import ( + "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" legacytranslator "github.com/aws/amazon-cloudwatch-agent/translator" "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) var nilSlice []string @@ -735,4 +740,4 @@ func TestTranslateAppSignals(t *testing.T) { } }) } -} \ No newline at end of file +} From ac0fa5f43dc59dd08a1ed23d64947991a31e1a32 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Wed, 25 Oct 2023 17:17:35 -0400 Subject: [PATCH 23/38] cleanup --- .github/workflows/PR-build.yml | 144 +++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 .github/workflows/PR-build.yml diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml new file mode 100644 index 0000000000..9bada91b1d --- /dev/null +++ b/.github/workflows/PR-build.yml @@ -0,0 +1,144 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +name: PR Build +on: + workflow_dispatch: + pull_request: + branches: + - main* + types: + - opened + - synchronize + - reopened + - ready_for_review + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + changes: + name: Check changes + runs-on: ubuntu-latest + outputs: + build: ${{ steps.filter.outputs.build }} + lint: ${{ steps.filter.outputs.lint }} + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v2 + id: filter + with: + list-files: shell + filters: .github/config/file-filters.yml + + - name: List all updated files + run: | + for file in ${{ steps.filter.outputs.build_files }}; do + echo "$file" + done + + lint: + needs: [changes] + name: Check lint + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.x + if: needs.changes.outputs.lint == 'true' + uses: actions/setup-go@v4 + with: + go-version: ~1.21.1 + cache: false + + - name: Check out code + if: needs.changes.outputs.lint == 'true' + uses: actions/checkout@v3 + + - name: Check format + if: needs.changes.outputs.lint == 'true' + run: | + make fmt fmt-sh + if [ ! -z "`git status --porcelain`" ]; then + echo "make fmt changed files" + git status + exit 1 + fi + + - name: Check license and imports + if: needs.changes.outputs.lint == 'true' + run: make simple-lint + + build: + needs: [lint, changes] + name: Build ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-2019, windows-latest, macos-11] + include: + - os: ubuntu-latest + family: linux + cache-path: | + ~/.cache/go-build + ~/go/pkg/mod + - os: macos-11 + family: darwin + cache-path: | + ~/Library/Caches/go-build + ~/go/pkg/mod + - os: windows-2019 + family: windows + cache-path: | + ~\AppData\Local\go-build + ~\go\pkg\mod + - os: windows-latest + family: windows + cache-path: | + ~\AppData\Local\go-build + ~\go\pkg\mod + steps: + - name: Set up Go 1.x + if: needs.changes.outputs.build == 'true' + uses: actions/setup-go@v4 + with: + go-version: ~1.21.1 + cache: false + + - name: Check out code + if: needs.changes.outputs.build == 'true' + uses: actions/checkout@v3 + + - name: Cache binaries + id: cached_binaries + if: needs.changes.outputs.build == 'true' + uses: actions/cache@v3 + with: + key: "cached-binaries-${{ matrix.os }}-${{ github.sha }}" + path: go.mod + + - name: Cache build output + if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + uses: actions/cache@v3 + with: + path: ${{ matrix.cache-path }} + key: v1-go-pkg-mod-${{ matrix.os }}-${{ hashFiles('**/go.sum') }} + + - name: Install make + if: matrix.family == 'windows' && steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + run: choco install make + + - name: Unit Test + if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + run: make test + + - name: Upload coverage to Codecov + if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + uses: codecov/codecov-action@v3 + with: + verbose: true + + - name: Build + if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' + run: make amazon-cloudwatch-agent-${{ matrix.family }} + \ No newline at end of file From 7cf1ebb29961c01acf5571a54845442d8a51c0fc Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Wed, 25 Oct 2023 17:18:05 -0400 Subject: [PATCH 24/38] spacing --- .github/workflows/PR-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml index 9bada91b1d..aebf5da7c0 100644 --- a/.github/workflows/PR-build.yml +++ b/.github/workflows/PR-build.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: pull_request: branches: - - main* + - main* types: - opened - synchronize @@ -141,4 +141,3 @@ jobs: - name: Build if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' run: make amazon-cloudwatch-agent-${{ matrix.family }} - \ No newline at end of file From bbe011f730fd0b8f68ffbc59f000c3f92bc33dbe Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Wed, 25 Oct 2023 17:19:34 -0400 Subject: [PATCH 25/38] revert PR-build.yml --- .github/workflows/PR-build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml index aebf5da7c0..8d178b2a4f 100644 --- a/.github/workflows/PR-build.yml +++ b/.github/workflows/PR-build.yml @@ -5,9 +5,9 @@ name: PR Build on: workflow_dispatch: pull_request: - branches: - - main* - types: + branches: + - main* + types: - opened - synchronize - reopened @@ -141,3 +141,4 @@ jobs: - name: Build if: steps.cached_binaries.outputs.cache-hit != 'true' && needs.changes.outputs.build == 'true' run: make amazon-cloudwatch-agent-${{ matrix.family }} + \ No newline at end of file From e753620434aa9ef9d163bdbb7a8d4b01d4f084db Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Wed, 25 Oct 2023 17:22:23 -0400 Subject: [PATCH 26/38] update resourcedetection processor translator --- .../translate/otel/processor/resourcedetection/translator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/translator/translate/otel/processor/resourcedetection/translator.go b/translator/translate/otel/processor/resourcedetection/translator.go index 485eb92020..9ac725b4dd 100644 --- a/translator/translate/otel/processor/resourcedetection/translator.go +++ b/translator/translate/otel/processor/resourcedetection/translator.go @@ -15,7 +15,7 @@ import ( ) //go:embed configs/config.yaml -var apmAwsResourceDetectionConfig string +var appSignalsAwsResourceDetectionConfig string type translator struct { name string @@ -57,5 +57,5 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*resourcedetectionprocessor.Config) - return common.GetYamlFileToYamlConfig(cfg, apmAwsResourceDetectionConfig) + return common.GetYamlFileToYamlConfig(cfg, appSignalsAwsResourceDetectionConfig) } From 3f8428098707bc0fd6c886f6ab22b79bc94edcfc Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Thu, 26 Oct 2023 12:17:04 -0400 Subject: [PATCH 27/38] Fix translation unit test failures --- .../sampleConfig/base_appsignals_config.yaml | 2 ++ .../otel/exporter/awsemf/appsignals_config_eks.yaml | 1 + .../exporter/awsemf/appsignals_config_generic.yaml | 1 + .../translate/otel/exporter/awsxray/translator_test.go | 1 + .../translate/otel/pipeline/appsignals/translator.go | 10 +++++----- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml index e727fc6fb1..a703e3c01f 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml @@ -65,6 +65,7 @@ exporters: - Fault - Error metric_descriptors: [] + middleware: agenthealth/logs namespace: AppSignals no_verify_ssl: false num_workers: 8 @@ -99,6 +100,7 @@ exporters: - HostedIn.Environment local_mode: false max_retries: 2 + middleware: agenthealth/traces no_verify_ssl: false num_workers: 8 profile: "" diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml index 19d72af94d..594116bdfb 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_eks.yaml @@ -1,5 +1,6 @@ log_group_name: "/aws/appsignals/eks" namespace: "AppSignals" +middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: diff --git a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml index bba205d0ac..d447430044 100644 --- a/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml +++ b/translator/translate/otel/exporter/awsemf/appsignals_config_generic.yaml @@ -1,5 +1,6 @@ log_group_name: "/aws/appsignals/generic" namespace: "AppSignals" +middleware: agenthealth/logs dimension_rollup_option: "NoDimensionRollup" metric_declarations: - dimensions: diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index 5966938b0d..a3ae0847d6 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -78,6 +78,7 @@ func TestTranslator(t *testing.T) { "enabled": true, "include_metadata": true, }, + "middleware": "agenthealth/traces", }), }, } diff --git a/translator/translate/otel/pipeline/appsignals/translator.go b/translator/translate/otel/pipeline/appsignals/translator.go index 15928c857f..c8face11c6 100644 --- a/translator/translate/otel/pipeline/appsignals/translator.go +++ b/translator/translate/otel/pipeline/appsignals/translator.go @@ -19,7 +19,7 @@ import ( ) const ( - pipelineName = "app_signals" + pipelineName = "appsignals" ) type translator struct { @@ -48,17 +48,17 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } translators := &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap(otlp.NewTranslatorWithName(common.AppSignals, otlp.WithDataType(t.dataType))), + Receivers: common.NewTranslatorMap(otlp.NewTranslatorWithName(pipelineName, otlp.WithDataType(t.dataType))), Processors: common.NewTranslatorMap(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType)), awsappsignals.NewTranslator(awsappsignals.WithDataType(t.dataType))), Exporters: common.NewTranslatorMap[component.Config](), Extensions: common.NewTranslatorMap[component.Config](), } if t.dataType == component.DataTypeTraces { - translators.Exporters.Set(awsxray.NewTranslatorWithName(common.AppSignals)) - translators.Extensions.Set(awsproxy.NewTranslatorWithName(common.AppSignals)) + translators.Exporters.Set(awsxray.NewTranslatorWithName(pipelineName)) + translators.Extensions.Set(awsproxy.NewTranslatorWithName(pipelineName)) } else { - translators.Exporters.Set(awsemf.NewTranslatorWithName(common.AppSignals)) + translators.Exporters.Set(awsemf.NewTranslatorWithName(pipelineName)) } return translators, nil } From 79e70157d15485ecb9e71ea66d817260d980daa9 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Thu, 26 Oct 2023 14:24:17 -0400 Subject: [PATCH 28/38] Fix translator again --- .../appsignals_and_kubernetes_config.yaml | 35 ++++++++++++------- .../sampleConfig/base_appsignals_config.yaml | 22 ++++++------ .../otel/pipeline/appsignals/translator.go | 14 +++----- .../pipeline/appsignals/translator_test.go | 14 ++++---- 4 files changed, 45 insertions(+), 40 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml index 031c9b7580..95d3f4bdf6 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml @@ -1,6 +1,6 @@ connectors: {} exporters: - awsemf/appsignals: + awsemf/app_signals: certificate_file_path: "" detailed_metrics: false dimension_rollup_option: NoDimensionRollup @@ -105,6 +105,7 @@ exporters: - Fault - Error metric_descriptors: [] + middleware: agenthealth/logs namespace: AppSignals no_verify_ssl: false num_workers: 8 @@ -222,6 +223,7 @@ exporters: - cluster_node_count - cluster_failed_node_count metric_descriptors: [] + middleware: agenthealth/logs namespace: ContainerInsights no_verify_ssl: false num_workers: 8 @@ -240,7 +242,7 @@ exporters: role_arn: "" shared_credentials_file: [] version: "0" - awsxray/appsignals: + awsxray/app_signals: aws_log_groups: [] certificate_file_path: "" endpoint: "" @@ -258,6 +260,7 @@ exporters: - HostedIn.Environment local_mode: false max_retries: 2 + middleware: agenthealth/traces no_verify_ssl: false num_workers: 8 profile: "" @@ -271,13 +274,18 @@ exporters: enabled: true include_metadata: true extensions: - awsproxy/appsignals: + awsproxy/app_signals: aws_endpoint: "" endpoint: 0.0.0.0:2000 local_mode: false proxy_address: "" region: "" role_arn: "" + agenthealth/logs: + is_usage_data_enabled: true + stats: + operations: + - PutLogEvents processors: awsappsignals: resolvers: @@ -575,7 +583,7 @@ receivers: resource_arn: "" role_arn: "" shared_credentials_file: [] - otlp/appsignals: + otlp/app_signals: protocols: grpc: auth: null @@ -601,16 +609,17 @@ receivers: traces_url_path: /v1/traces service: extensions: - - awsproxy/apm + - awsproxy/app_signals + - agenthealth/logs pipelines: - metrics/apm: + metrics/app_signals: exporters: - - awsemf/apm + - awsemf/app_signals processors: - resourcedetection - - awsapm + - awsappsignals receivers: - - otlp/apm + - otlp/app_signals metrics/containerinsights: exporters: - awsemf/containerinsights @@ -618,14 +627,14 @@ service: - batch/containerinsights receivers: - awscontainerinsightreceiver - traces/apm: + traces/app_signals: exporters: - - awsxray/apm + - awsxray/app_signals processors: - resourcedetection - - awsapm + - awsappsignals receivers: - - otlp/apm + - otlp/app_signals telemetry: logs: development: false diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml index a703e3c01f..97db0160aa 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml @@ -1,6 +1,6 @@ connectors: {} exporters: - awsemf/appsignals: + awsemf/app_signals: certificate_file_path: "" detailed_metrics: false dimension_rollup_option: NoDimensionRollup @@ -82,7 +82,7 @@ exporters: role_arn: "" shared_credentials_file: [] version: "1" - awsxray/appsignals: + awsxray/app_signals: aws_log_groups: [] certificate_file_path: "" endpoint: https://fake_endpoint @@ -114,7 +114,7 @@ exporters: enabled: true include_metadata: true extensions: - awsproxy/appsignals: + awsproxy/app_signals: aws_endpoint: "" endpoint: 0.0.0.0:2000 local_mode: false @@ -387,7 +387,7 @@ processors: timeout: 2s write_buffer_size: 0 receivers: - otlp/appsignals: + otlp/app_signals: protocols: grpc: auth: null @@ -413,24 +413,24 @@ receivers: traces_url_path: /v1/traces service: extensions: - - awsproxy/appsignals + - awsproxy/app_signals pipelines: - metrics/appsignals: + metrics/app_signals: exporters: - - awsemf/appsignals + - awsemf/app_signals processors: - resourcedetection - awsappsignals receivers: - - otlp/appsignals - traces/appsignals: + - otlp/app_signals + traces/app_signals: exporters: - - awsxray/appsignals + - awsxray/app_signals processors: - resourcedetection - awsappsignals receivers: - - otlp/appsignals + - otlp/app_signals telemetry: logs: development: false diff --git a/translator/translate/otel/pipeline/appsignals/translator.go b/translator/translate/otel/pipeline/appsignals/translator.go index c8face11c6..05efd2ea43 100644 --- a/translator/translate/otel/pipeline/appsignals/translator.go +++ b/translator/translate/otel/pipeline/appsignals/translator.go @@ -18,10 +18,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" ) -const ( - pipelineName = "appsignals" -) - type translator struct { dataType component.DataType } @@ -35,7 +31,7 @@ func NewTranslator(dataType component.DataType) common.Translator[*common.Compon } func (t *translator) ID() component.ID { - return component.NewIDWithName(t.dataType, pipelineName) + return component.NewIDWithName(t.dataType, common.AppSignals) } func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) { @@ -48,17 +44,17 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } translators := &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap(otlp.NewTranslatorWithName(pipelineName, otlp.WithDataType(t.dataType))), + Receivers: common.NewTranslatorMap(otlp.NewTranslatorWithName(common.AppSignals, otlp.WithDataType(t.dataType))), Processors: common.NewTranslatorMap(resourcedetection.NewTranslator(resourcedetection.WithDataType(t.dataType)), awsappsignals.NewTranslator(awsappsignals.WithDataType(t.dataType))), Exporters: common.NewTranslatorMap[component.Config](), Extensions: common.NewTranslatorMap[component.Config](), } if t.dataType == component.DataTypeTraces { - translators.Exporters.Set(awsxray.NewTranslatorWithName(pipelineName)) - translators.Extensions.Set(awsproxy.NewTranslatorWithName(pipelineName)) + translators.Exporters.Set(awsxray.NewTranslatorWithName(common.AppSignals)) + translators.Extensions.Set(awsproxy.NewTranslatorWithName(common.AppSignals)) } else { - translators.Exporters.Set(awsemf.NewTranslatorWithName(pipelineName)) + translators.Exporters.Set(awsemf.NewTranslatorWithName(common.AppSignals)) } return translators, nil } diff --git a/translator/translate/otel/pipeline/appsignals/translator_test.go b/translator/translate/otel/pipeline/appsignals/translator_test.go index 3d60640714..91afbffa55 100644 --- a/translator/translate/otel/pipeline/appsignals/translator_test.go +++ b/translator/translate/otel/pipeline/appsignals/translator_test.go @@ -24,7 +24,7 @@ func TestTranslatorTraces(t *testing.T) { extensions []string } tt := NewTranslator(component.DataTypeTraces) - assert.EqualValues(t, "traces/appsignals", tt.ID().String()) + assert.EqualValues(t, "traces/app_signals", tt.ID().String()) testCases := map[string]struct { input map[string]interface{} want *want @@ -43,10 +43,10 @@ func TestTranslatorTraces(t *testing.T) { }, }, want: &want{ - receivers: []string{"otlp/appsignals"}, + receivers: []string{"otlp/app_signals"}, processors: []string{"resourcedetection", "awsappsignals"}, - exporters: []string{"awsxray/appsignals"}, - extensions: []string{"awsproxy/appsignals"}, + exporters: []string{"awsxray/app_signals"}, + extensions: []string{"awsproxy/app_signals"}, }, }, } @@ -74,7 +74,7 @@ func TestTranslatorMetrics(t *testing.T) { exporters []string } tt := NewTranslator(component.DataTypeMetrics) - assert.EqualValues(t, "metrics/appsignals", tt.ID().String()) + assert.EqualValues(t, "metrics/app_signals", tt.ID().String()) testCases := map[string]struct { input map[string]interface{} want *want @@ -93,9 +93,9 @@ func TestTranslatorMetrics(t *testing.T) { }, }, want: &want{ - receivers: []string{"otlp/appsignals"}, + receivers: []string{"otlp/app_signals"}, processors: []string{"resourcedetection", "awsappsignals"}, - exporters: []string{"awsemf/appsignals"}, + exporters: []string{"awsemf/app_signals"}, }, }, } From 15c49026699f0c9d12600013ed3c07fc65f7679d Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Thu, 26 Oct 2023 19:43:58 -0400 Subject: [PATCH 29/38] fix eks resolver unit test --- .../internal/resolver/eks_test.go | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/plugins/processors/awsappsignals/internal/resolver/eks_test.go b/plugins/processors/awsappsignals/internal/resolver/eks_test.go index 7da36d9c99..ed107d1329 100644 --- a/plugins/processors/awsappsignals/internal/resolver/eks_test.go +++ b/plugins/processors/awsappsignals/internal/resolver/eks_test.go @@ -802,17 +802,7 @@ func TestEksResolver(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "not-an-ip", getStrAttr(attributes, "aws.remote.service", t)) - // Test case 4 and 5: resourceAttributes contains "k8s.namespace.name" and EKS cluster name - attributes = pcommon.NewMap() - resourceAttributes = pcommon.NewMap() - resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") - resourceAttributes.PutStr("ec2.tag.kubernetes.io/cluster/test-cluster", "owned") - err = resolver.Process(attributes, resourceAttributes) - assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, "HostedIn.K8s.Namespace", t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, "HostedIn.EKS.Cluster", t)) - - // Test case 6: Process with valid IP but GetWorkloadAndNamespaceByIP returns error + // Test case 4: Process with valid IP but GetWorkloadAndNamespaceByIP returns error attributes = pcommon.NewMap() attributes.PutStr("aws.remote.service", "192.168.1.2") resourceAttributes = pcommon.NewMap() @@ -822,6 +812,31 @@ func TestEksResolver(t *testing.T) { }) } +func TestHostedInEksResolver(t *testing.T) { + // helper function to get string values from the attributes + getStrAttr := func(attributes pcommon.Map, key string, t *testing.T) string { + if value, ok := attributes.Get(key); ok { + return value.AsString() + } else { + t.Errorf("Failed to get value for key: %s", key) + return "" + } + } + + resolver := newEKSHostedInAttributeResolver() + + // Test case 1 and 2: resourceAttributes contains "k8s.namespace.name" and EKS cluster name + attributes := pcommon.NewMap() + resourceAttributes := pcommon.NewMap() + resourceAttributes.PutStr("cloud.provider", "aws") + resourceAttributes.PutStr("k8s.namespace.name", "test-namespace-3") + resourceAttributes.PutStr("ec2.tag.kubernetes.io/cluster/test-cluster", "owned") + err := resolver.Process(attributes, resourceAttributes) + assert.NoError(t, err) + assert.Equal(t, "test-namespace-3", getStrAttr(attributes, "HostedIn.K8s.Namespace", t)) + assert.Equal(t, "test-cluster", getStrAttr(attributes, "HostedIn.EKS.Cluster", t)) +} + func TestExtractIPPort(t *testing.T) { // Test valid IP:Port ip, port, ok := extractIPPort("192.0.2.0:8080") From 208f25f150847dcf5fc40e9b571bbfdba92000fa Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 27 Oct 2023 14:48:06 -0400 Subject: [PATCH 30/38] Add agenthealth/traces to appsignals translator. Create internal attributes package in awsappsignals processor. Minor tweaks --- .../agenthealth/handler/stats/handler.go | 9 ++++ go.mod | 11 ++--- plugins/processors/awsappsignals/README.md | 4 +- plugins/processors/awsappsignals/factory.go | 3 +- .../internal/attributes/attributes.go | 28 ++++++++++++ .../normalizer/attributesnormalizer.go | 44 ++++++++++--------- .../internal/resolver/attributesresolver.go | 14 +++--- .../awsappsignals/internal/resolver/eks.go | 22 +++++----- .../internal/resolver/eks_test.go | 26 ++++++----- .../awsappsignals/internal/resolver/types.go | 18 -------- plugins/processors/awsappsignals/processor.go | 19 ++++---- .../otel/pipeline/appsignals/translator.go | 3 ++ .../pipeline/appsignals/translator_test.go | 6 ++- 13 files changed, 121 insertions(+), 86 deletions(-) create mode 100644 plugins/processors/awsappsignals/internal/attributes/attributes.go delete mode 100644 plugins/processors/awsappsignals/internal/resolver/types.go diff --git a/extension/agenthealth/handler/stats/handler.go b/extension/agenthealth/handler/stats/handler.go index f5fe991125..e4bc0299e4 100644 --- a/extension/agenthealth/handler/stats/handler.go +++ b/extension/agenthealth/handler/stats/handler.go @@ -5,6 +5,7 @@ package stats import ( "context" + "log" "net/http" "sync" @@ -63,6 +64,14 @@ func (sh *statsHandler) HandleRequest(ctx context.Context, r *http.Request) { header := sh.Header(operation) if header != "" { r.Header.Set(headerKeyAgentStats, header) + + request := awsmiddleware.GetRequestID(ctx) + if header != "" { + r.Header.Set(headerKeyAgentStats, header) + log.Printf("D! stats header (%s/%s): %s", operation, request, header) + } else { + log.Printf("W! empty stats header (%s/%s)", operation, request) + } } } diff --git a/go.mod b/go.mod index 92a1c9c47b..e59ce1e3c8 100644 --- a/go.mod +++ b/go.mod @@ -102,6 +102,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/eks v1.27.15 github.com/aws/smithy-go v1.15.0 github.com/bigkevmcd/go-configparser v0.0.0-20200217161103-d137835d2579 + github.com/deckarep/golang-set/v2 v2.3.1 github.com/go-kit/log v0.2.1 github.com/gobwas/glob v0.2.3 github.com/google/cadvisor v0.47.3 // indirect @@ -117,10 +118,12 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.84.0 + github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.84.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.84.0 @@ -147,6 +150,7 @@ require ( go.opentelemetry.io/collector/processor/batchprocessor v0.84.1-0.20230908201109-ab3d6c5b6470 go.opentelemetry.io/collector/receiver v0.84.1-0.20230908201109-ab3d6c5b6470 go.opentelemetry.io/collector/receiver/otlpreceiver v0.84.0 + go.opentelemetry.io/collector/semconv v0.84.1-0.20230908201109-ab3d6c5b6470 go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.25.0 @@ -166,13 +170,6 @@ require ( k8s.io/klog/v2 v2.100.1 ) -require ( - github.com/deckarep/golang-set/v2 v2.3.1 - github.com/open-telemetry/opentelemetry-collector-contrib/extension/awsproxy v0.84.0 - github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.84.0 - go.opentelemetry.io/collector/semconv v0.84.1-0.20230908201109-ab3d6c5b6470 -) - require ( cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 // indirect diff --git a/plugins/processors/awsappsignals/README.md b/plugins/processors/awsappsignals/README.md index f270a3a734..4906b69a46 100644 --- a/plugins/processors/awsappsignals/README.md +++ b/plugins/processors/awsappsignals/README.md @@ -1,7 +1,7 @@ # AWS AppSignals Processor for Amazon Cloudwatch Agent -The AWS AppSignals processor is used to reduce the cardinality of telemtry metrics and traces before exporting them to CloudWatch Logs via [EMF](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awsemfexporter) and [X-Ray](github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter) respectively. -It reduces the cardinality of metrics/traces via 3 types of actions, `keep`, `drop` and `replace`, which are configured by users. CloudWatch Agent(CWA) customers will configure these rules with their CWA configurations, which will get translated to yaml. +The AWS AppSignals processor is used to reduce the cardinality of telemetry metrics and traces before exporting them to CloudWatch Logs via [EMF](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awsemfexporter) and [X-Ray](github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter) respectively. +It reduces the cardinality of metrics/traces via 3 types of actions, `keep`, `drop` and `replace`, which are configured by users. CloudWatch Agent(CWA) customers will configure these rules with their CWA configurations. Note: Traces support only `replace` actions and are implicitly pulled from the logs section of the CWA configuration diff --git a/plugins/processors/awsappsignals/factory.go b/plugins/processors/awsappsignals/factory.go index c00150a85a..8cb3bc6298 100644 --- a/plugins/processors/awsappsignals/factory.go +++ b/plugins/processors/awsappsignals/factory.go @@ -34,7 +34,8 @@ func NewFactory() processor.Factory { func createDefaultConfig() component.Config { return &Config{ - Resolvers: []string{"eks"}, // we need to change the default config later + // TODO: change default config when other resolvers are supported + Resolvers: []string{"eks"}, } } diff --git a/plugins/processors/awsappsignals/internal/attributes/attributes.go b/plugins/processors/awsappsignals/internal/attributes/attributes.go new file mode 100644 index 0000000000..bc7c1eb3eb --- /dev/null +++ b/plugins/processors/awsappsignals/internal/attributes/attributes.go @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package attributes + +const ( + // aws attributes + AWSLocalService = "aws.local.service" + AWSLocalOperation = "aws.local.operation" + AWSRemoteService = "aws.remote.service" + AWSRemoteOperation = "aws.remote.operation" + AWSRemoteTarget = "aws.remote.target" + AWSHostedInEnvironment = "aws.hostedin.environment" + + // kubernetes resource attributes + K8SDeploymentName = "k8s.deployment.name" + K8SStatefulSetName = "k8s.statefulset.name" + K8SDaemonSetName = "k8s.daemonset.name" + K8SJobName = "k8s.job.name" + K8SCronJobName = "k8s.cronjob.name" + K8SPodName = "k8s.pod.name" + K8SRemoteNamespace = "K8s.RemoteNamespace" + + // hosted in attribute names + HostedInClusterName = "HostedIn.EKS.Cluster" + HostedInK8SNamespace = "HostedIn.K8s.Namespace" + HostedInEnvironment = "HostedIn.Environment" +) diff --git a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go index a72e73f01c..25a6af641e 100644 --- a/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go +++ b/plugins/processors/awsappsignals/internal/normalizer/attributesnormalizer.go @@ -6,6 +6,8 @@ package normalizer import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" + + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" ) type attributesNormalizer struct { @@ -13,24 +15,24 @@ type attributesNormalizer struct { } var renameMapForMetric = map[string]string{ - "aws.local.service": "Service", - "aws.local.operation": "Operation", - "aws.remote.service": "RemoteService", - "aws.remote.operation": "RemoteOperation", - "aws.remote.target": "RemoteTarget", + attr.AWSLocalService: "Service", + attr.AWSLocalOperation: "Operation", + attr.AWSRemoteService: "RemoteService", + attr.AWSRemoteOperation: "RemoteOperation", + attr.AWSRemoteTarget: "RemoteTarget", } var renameMapForTrace = map[string]string{ - // these kubernetes resource attributes are set by the openTelemtry operator - // see the code referecnes from upstream: + // these kubernetes resource attributes are set by the openTelemetry operator + // see the code references from upstream: // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 - "k8s.deployment.name": "K8s.Workload", - "k8s.statefulset.name": "K8s.Workload", - "k8s.daemonset.name": "K8s.Workload", - "k8s.job.name": "K8s.Workload", - "k8s.cronjob.name": "K8s.Workload", - "k8s.pod.name": "K8s.Pod", + attr.K8SDeploymentName: "K8s.Workload", + attr.K8SStatefulSetName: "K8s.Workload", + attr.K8SDaemonSetName: "K8s.Workload", + attr.K8SJobName: "K8s.Workload", + attr.K8SCronJobName: "K8s.Workload", + attr.K8SPodName: "K8s.Pod", } var copyMapForMetric = map[string]string{ @@ -38,12 +40,12 @@ var copyMapForMetric = map[string]string{ // see the code referecnes from upstream: // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L245 // * https://github.com/open-telemetry/opentelemetry-operator/blob/0e39ee77693146e0924da3ca474a0fe14dc30b3a/pkg/instrumentation/sdk.go#L305C43-L305C43 - "k8s.deployment.name": "K8s.Workload", - "k8s.statefulset.name": "K8s.Workload", - "k8s.daemonset.name": "K8s.Workload", - "k8s.job.name": "K8s.Workload", - "k8s.cronjob.name": "K8s.Workload", - "k8s.pod.name": "K8s.Pod", + attr.K8SDeploymentName: "K8s.Workload", + attr.K8SStatefulSetName: "K8s.Workload", + attr.K8SDaemonSetName: "K8s.Workload", + attr.K8SJobName: "K8s.Workload", + attr.K8SCronJobName: "K8s.Workload", + attr.K8SPodName: "K8s.Pod", } func NewAttributesNormalizer(logger *zap.Logger) *attributesNormalizer { @@ -80,7 +82,7 @@ func (n *attributesNormalizer) copyResourceAttributesToAttributes(attributes, re n.logger.Debug("attribute value is overwritten", zap.String("attribute", k), zap.String("original", originalAttrValue.AsString()), zap.String("new", resourceAttrValue.AsString())) } attributes.PutStr(v, resourceAttrValue.AsString()) - if k == "k8s.pod.name" { + if k == attr.K8SPodName { // only copy "host.id" from resource attributes to "K8s.Node" in attributesif the pod name is set if host, ok := resourceAttributes.Get("host.id"); ok { attributes.PutStr("K8s.Node", host.AsString()) @@ -95,7 +97,7 @@ func rename(attrs pcommon.Map, renameMap map[string]string) { if value, ok := attrs.Get(original); ok { attrs.PutStr(replacement, value.AsString()) attrs.Remove(original) - if original == "k8s.pod.name" { + if original == attr.K8SPodName { // only rename host.id if the pod name is set if host, ok := attrs.Get("host.id"); ok { attrs.PutStr("K8s.Node", host.AsString()) diff --git a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go index 0b4525d38a..454c0f679a 100644 --- a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go @@ -5,13 +5,16 @@ package resolver import ( "context" + "errors" "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" + + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" ) var DefaultHostedInAttributes = map[string]string{ - AttributeHostedInEnvironment: HostedInAttributeEnvironment, + attr.AWSHostedInEnvironment: attr.HostedInEnvironment, } type subResolver interface { @@ -49,12 +52,13 @@ func (r *attributesResolver) Process(attributes, resourceAttributes pcommon.Map, } func (r *attributesResolver) Stop(ctx context.Context) error { + var errs error for _, subResolver := range r.subResolvers { if err := subResolver.Stop(ctx); err != nil { - return err + errs = errors.Join(errs, err) } } - return nil + return errs } type hostedInAttributeResolver struct { @@ -73,9 +77,9 @@ func (h *hostedInAttributeResolver) Process(attributes, resourceAttributes pcomm } } - if _, ok := resourceAttributes.Get(AttributeHostedInEnvironment); !ok { + if _, ok := resourceAttributes.Get(attr.HostedInEnvironment); !ok { hostedInEnv := "Generic" - attributes.PutStr(HostedInAttributeEnvironment, hostedInEnv) + attributes.PutStr(attr.HostedInEnvironment, hostedInEnv) } return nil diff --git a/plugins/processors/awsappsignals/internal/resolver/eks.go b/plugins/processors/awsappsignals/internal/resolver/eks.go index 6f65f24f2e..af04b42ee1 100644 --- a/plugins/processors/awsappsignals/internal/resolver/eks.go +++ b/plugins/processors/awsappsignals/internal/resolver/eks.go @@ -24,6 +24,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" ) const ( @@ -44,7 +46,7 @@ const ( ) var DefaultHostedInAttributeMap = map[string]string{ - semconv.AttributeK8SNamespaceName: HostedInAttributeK8SNamespace, + semconv.AttributeK8SNamespaceName: attr.HostedInClusterName, } var ( @@ -625,13 +627,13 @@ func (e *eksResolver) debug() { } func (e *eksResolver) Process(attributes, resourceAttributes pcommon.Map) error { - if value, ok := attributes.Get(AttributeRemoteService); ok { + if value, ok := attributes.Get(attr.AWSRemoteService); ok { valueStr := value.AsString() ipStr := "" if ip, _, ok := extractIPPort(valueStr); ok { if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(valueStr); err == nil { - attributes.PutStr(AttributeRemoteService, workload) - attributes.PutStr(AttributeRemoteNamespace, namespace) + attributes.PutStr(attr.AWSRemoteService, workload) + attributes.PutStr(attr.K8SRemoteNamespace, namespace) } else { ipStr = ip } @@ -641,11 +643,11 @@ func (e *eksResolver) Process(attributes, resourceAttributes pcommon.Map) error if ipStr != "" { if workload, namespace, err := e.GetWorkloadAndNamespaceByIP(ipStr); err == nil { - attributes.PutStr(AttributeRemoteService, workload) - attributes.PutStr(AttributeRemoteNamespace, namespace) + attributes.PutStr(attr.AWSRemoteService, workload) + attributes.PutStr(attr.K8SRemoteNamespace, namespace) } else { e.logger.Debug("failed to Process ip", zap.String("ip", ipStr), zap.Error(err)) - attributes.PutStr(AttributeRemoteService, "UnknownRemoteService") + attributes.PutStr(attr.AWSRemoteService, "UnknownRemoteService") } } } @@ -703,7 +705,7 @@ type eksHostedInAttributeResolver struct { func newEKSHostedInAttributeResolver() *eksHostedInAttributeResolver { return &eksHostedInAttributeResolver{ attributeMap: map[string]string{ - semconv.AttributeK8SNamespaceName: HostedInAttributeK8SNamespace, + semconv.AttributeK8SNamespaceName: attr.HostedInK8SNamespace, }, } } @@ -715,7 +717,7 @@ func (h *eksHostedInAttributeResolver) Process(attributes, resourceAttributes pc } if h.clusterName != "" { - attributes.PutStr(HostedInAttributeClusterName, h.clusterName) + attributes.PutStr(attr.HostedInClusterName, h.clusterName) } else { platform, _ := resourceAttributes.Get(semconv.AttributeCloudProvider) if platform.AsString() == semconv.AttributeCloudProviderAWS { @@ -723,7 +725,7 @@ func (h *eksHostedInAttributeResolver) Process(attributes, resourceAttributes pc resourceAttributes.Range(func(key string, value pcommon.Value) bool { if strings.HasPrefix(key, "ec2.tag.kubernetes.io/cluster/") && value.Type() == pcommon.ValueTypeStr && value.AsString() == "owned" { h.clusterName = strings.TrimPrefix(key, "ec2.tag.kubernetes.io/cluster/") - attributes.PutStr(HostedInAttributeClusterName, h.clusterName) + attributes.PutStr(attr.HostedInClusterName, h.clusterName) return false } return true diff --git a/plugins/processors/awsappsignals/internal/resolver/eks_test.go b/plugins/processors/awsappsignals/internal/resolver/eks_test.go index ed107d1329..5f92a55f80 100644 --- a/plugins/processors/awsappsignals/internal/resolver/eks_test.go +++ b/plugins/processors/awsappsignals/internal/resolver/eks_test.go @@ -16,6 +16,8 @@ import ( "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + attr "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/attributes" ) // MockDeleter deletes a key immediately, useful for testing. @@ -774,41 +776,41 @@ func TestEksResolver(t *testing.T) { // Test case 1: "aws.remote.service" contains IP:Port attributes := pcommon.NewMap() - attributes.PutStr("aws.remote.service", "192.0.2.1:8080") + attributes.PutStr(attr.AWSRemoteService, "192.0.2.1:8080") resourceAttributes := pcommon.NewMap() resolver.ipToPod.Store("192.0.2.1:8080", "test-pod") resolver.podToWorkloadAndNamespace.Store("test-pod", "test-deployment@test-namespace") err := resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) - assert.Equal(t, "test-deployment", getStrAttr(attributes, "aws.remote.service", t)) - assert.Equal(t, "test-namespace", getStrAttr(attributes, "K8s.RemoteNamespace", t)) + assert.Equal(t, "test-deployment", getStrAttr(attributes, attr.AWSRemoteService, t)) + assert.Equal(t, "test-namespace", getStrAttr(attributes, attr.K8SRemoteNamespace, t)) // Test case 2: "aws.remote.service" contains only IP attributes = pcommon.NewMap() - attributes.PutStr("aws.remote.service", "192.0.2.2") + attributes.PutStr(attr.AWSRemoteService, "192.0.2.2") resourceAttributes = pcommon.NewMap() resolver.ipToPod.Store("192.0.2.2", "test-pod-2") resolver.podToWorkloadAndNamespace.Store("test-pod-2", "test-deployment-2@test-namespace-2") err = resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) - assert.Equal(t, "test-deployment-2", getStrAttr(attributes, "aws.remote.service", t)) - assert.Equal(t, "test-namespace-2", getStrAttr(attributes, "K8s.RemoteNamespace", t)) + assert.Equal(t, "test-deployment-2", getStrAttr(attributes, attr.AWSRemoteService, t)) + assert.Equal(t, "test-namespace-2", getStrAttr(attributes, attr.K8SRemoteNamespace, t)) // Test case 3: "aws.remote.service" contains non-ip string attributes = pcommon.NewMap() - attributes.PutStr("aws.remote.service", "not-an-ip") + attributes.PutStr(attr.AWSRemoteService, "not-an-ip") resourceAttributes = pcommon.NewMap() err = resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) - assert.Equal(t, "not-an-ip", getStrAttr(attributes, "aws.remote.service", t)) + assert.Equal(t, "not-an-ip", getStrAttr(attributes, attr.AWSRemoteService, t)) // Test case 4: Process with valid IP but GetWorkloadAndNamespaceByIP returns error attributes = pcommon.NewMap() - attributes.PutStr("aws.remote.service", "192.168.1.2") + attributes.PutStr(attr.AWSRemoteService, "192.168.1.2") resourceAttributes = pcommon.NewMap() err = resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) - assert.Equal(t, "UnknownRemoteService", getStrAttr(attributes, "aws.remote.service", t)) + assert.Equal(t, "UnknownRemoteService", getStrAttr(attributes, attr.AWSRemoteService, t)) }) } @@ -833,8 +835,8 @@ func TestHostedInEksResolver(t *testing.T) { resourceAttributes.PutStr("ec2.tag.kubernetes.io/cluster/test-cluster", "owned") err := resolver.Process(attributes, resourceAttributes) assert.NoError(t, err) - assert.Equal(t, "test-namespace-3", getStrAttr(attributes, "HostedIn.K8s.Namespace", t)) - assert.Equal(t, "test-cluster", getStrAttr(attributes, "HostedIn.EKS.Cluster", t)) + assert.Equal(t, "test-namespace-3", getStrAttr(attributes, attr.HostedInK8SNamespace, t)) + assert.Equal(t, "test-cluster", getStrAttr(attributes, attr.HostedInClusterName, t)) } func TestExtractIPPort(t *testing.T) { diff --git a/plugins/processors/awsappsignals/internal/resolver/types.go b/plugins/processors/awsappsignals/internal/resolver/types.go deleted file mode 100644 index dbaa8fa7c1..0000000000 --- a/plugins/processors/awsappsignals/internal/resolver/types.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package resolver - -const ( - AttributeRemoteService = "aws.remote.service" - AttributeHostedInEnvironment = "aws.hostedin.environment" -) - -const ( - AttributeRemoteNamespace = "K8s.RemoteNamespace" -) -const ( - HostedInAttributeClusterName = "HostedIn.EKS.Cluster" - HostedInAttributeK8SNamespace = "HostedIn.K8s.Namespace" - HostedInAttributeEnvironment = "HostedIn.Environment" -) diff --git a/plugins/processors/awsappsignals/processor.go b/plugins/processors/awsappsignals/processor.go index b4375dedc2..1c9e63cf04 100644 --- a/plugins/processors/awsappsignals/processor.go +++ b/plugins/processors/awsappsignals/processor.go @@ -18,8 +18,8 @@ import ( ) const ( - failedToProcessAttribute = "failed to Process attributes" - failedToProcessAttributeWithCustomRule = "failed to process attributes with custom rule, will drop it" + failedToProcessAttribute = "failed to process attributes" + failedToProcessAttributeWithCustomRule = "failed to process attributes with custom rule, will drop the metric" ) // this is used to Process some attributes (like IP addresses) to a generic form to reduce high cardinality @@ -47,19 +47,20 @@ type awsappsignalsprocessor struct { func (ap *awsappsignalsprocessor) Start(_ context.Context, _ component.Host) error { attributesResolver := resolver.NewAttributesResolver(ap.config.Resolvers, ap.logger) - ap.stoppers = append(ap.stoppers, attributesResolver) - ap.metricMutators = append(ap.metricMutators, attributesResolver) + ap.stoppers = []stopper{attributesResolver} + ap.metricMutators = []attributesMutator{attributesResolver} attributesNormalizer := normalizer.NewAttributesNormalizer(ap.logger) - ap.metricMutators = append(ap.metricMutators, attributesNormalizer) + ap.metricMutators = []attributesMutator{attributesResolver, attributesNormalizer} ap.customReplacer = customconfiguration.NewCustomReplacer(ap.config.Rules) - ap.traceMutators = append(ap.traceMutators, attributesResolver, attributesNormalizer, ap.customReplacer) + ap.traceMutators = []attributesMutator{attributesResolver, attributesNormalizer, ap.customReplacer} customKeeper := customconfiguration.NewCustomKeeper(ap.config.Rules) - ap.customAllowlistMutators = append(ap.customAllowlistMutators, customKeeper) + ap.customAllowlistMutators = []customAllowlistMutator{customKeeper} + customDropper := customconfiguration.NewCustomDropper(ap.config.Rules) - ap.customAllowlistMutators = append(ap.customAllowlistMutators, customDropper) + ap.customAllowlistMutators = []customAllowlistMutator{customDropper} return nil } @@ -263,6 +264,6 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(ctx context.Context, m } } default: - ap.logger.Info("Ignore unknown metric type", zap.String("type", m.Type().String())) + ap.logger.Debug("Ignore unknown metric type", zap.String("type", m.Type().String())) } } diff --git a/translator/translate/otel/pipeline/appsignals/translator.go b/translator/translate/otel/pipeline/appsignals/translator.go index 05efd2ea43..71aafb8d92 100644 --- a/translator/translate/otel/pipeline/appsignals/translator.go +++ b/translator/translate/otel/pipeline/appsignals/translator.go @@ -12,6 +12,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsxray" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/awsproxy" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/awsappsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" @@ -53,8 +54,10 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators if t.dataType == component.DataTypeTraces { translators.Exporters.Set(awsxray.NewTranslatorWithName(common.AppSignals)) translators.Extensions.Set(awsproxy.NewTranslatorWithName(common.AppSignals)) + translators.Extensions.Set(agenthealth.NewTranslator(component.DataTypeTraces, []string{agenthealth.OperationPutTraceSegments})) } else { translators.Exporters.Set(awsemf.NewTranslatorWithName(common.AppSignals)) + translators.Extensions.Set(agenthealth.NewTranslator(component.DataTypeLogs, []string{agenthealth.OperationPutLogEvents})) } return translators, nil } diff --git a/translator/translate/otel/pipeline/appsignals/translator_test.go b/translator/translate/otel/pipeline/appsignals/translator_test.go index 91afbffa55..2a250903a5 100644 --- a/translator/translate/otel/pipeline/appsignals/translator_test.go +++ b/translator/translate/otel/pipeline/appsignals/translator_test.go @@ -46,7 +46,7 @@ func TestTranslatorTraces(t *testing.T) { receivers: []string{"otlp/app_signals"}, processors: []string{"resourcedetection", "awsappsignals"}, exporters: []string{"awsxray/app_signals"}, - extensions: []string{"awsproxy/app_signals"}, + extensions: []string{"awsproxy/app_signals", "agenthealth/traces"}, }, }, } @@ -62,6 +62,7 @@ func TestTranslatorTraces(t *testing.T) { assert.Equal(t, testCase.want.receivers, collections.MapSlice(got.Receivers.Keys(), component.ID.String)) assert.Equal(t, testCase.want.processors, collections.MapSlice(got.Processors.Keys(), component.ID.String)) assert.Equal(t, testCase.want.exporters, collections.MapSlice(got.Exporters.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.extensions, collections.MapSlice(got.Extensions.Keys(), component.ID.String)) } }) } @@ -72,6 +73,7 @@ func TestTranslatorMetrics(t *testing.T) { receivers []string processors []string exporters []string + extensions []string } tt := NewTranslator(component.DataTypeMetrics) assert.EqualValues(t, "metrics/app_signals", tt.ID().String()) @@ -96,6 +98,7 @@ func TestTranslatorMetrics(t *testing.T) { receivers: []string{"otlp/app_signals"}, processors: []string{"resourcedetection", "awsappsignals"}, exporters: []string{"awsemf/app_signals"}, + extensions: []string{"agenthealth/logs"}, }, }, } @@ -111,6 +114,7 @@ func TestTranslatorMetrics(t *testing.T) { assert.Equal(t, testCase.want.receivers, collections.MapSlice(got.Receivers.Keys(), component.ID.String)) assert.Equal(t, testCase.want.processors, collections.MapSlice(got.Processors.Keys(), component.ID.String)) assert.Equal(t, testCase.want.exporters, collections.MapSlice(got.Exporters.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.extensions, collections.MapSlice(got.Extensions.Keys(), component.ID.String)) } }) } From 26a739c47e8763f3ebe980c620421355c953f8ba Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 27 Oct 2023 14:51:55 -0400 Subject: [PATCH 31/38] Update otel-fork-replace workflow --- .github/workflows/otel-fork-replace.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/otel-fork-replace.yml b/.github/workflows/otel-fork-replace.yml index eb56e137d7..553eed7d48 100644 --- a/.github/workflows/otel-fork-replace.yml +++ b/.github/workflows/otel-fork-replace.yml @@ -38,6 +38,10 @@ jobs: git config --global user.name 'Github Action' git config --global user.email 'action@github.com' git checkout -b otel-fork-replace-${{ steps.get-latest-commit.outputs.sha }} + go mod edit -replace go.opentelemetry.io/collector/config/confighttp=github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp@${{ steps.get-latest-commit.outputs.sha }} + go mod tidy + go mod edit -replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor=github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor@${{ steps.get-latest-commit.outputs.sha }} + go mod tidy go mod edit -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter=github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter@${{ steps.get-latest-commit.outputs.sha }} go mod tidy go mod edit -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter=github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter@${{ steps.get-latest-commit.outputs.sha }} From 2212000c11ff1b4f0e033dcf546a4a4105ad79eb Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 27 Oct 2023 14:55:34 -0400 Subject: [PATCH 32/38] Revert agenthealth handler to main --- extension/agenthealth/handler/stats/handler.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/extension/agenthealth/handler/stats/handler.go b/extension/agenthealth/handler/stats/handler.go index e4bc0299e4..f5fe991125 100644 --- a/extension/agenthealth/handler/stats/handler.go +++ b/extension/agenthealth/handler/stats/handler.go @@ -5,7 +5,6 @@ package stats import ( "context" - "log" "net/http" "sync" @@ -64,14 +63,6 @@ func (sh *statsHandler) HandleRequest(ctx context.Context, r *http.Request) { header := sh.Header(operation) if header != "" { r.Header.Set(headerKeyAgentStats, header) - - request := awsmiddleware.GetRequestID(ctx) - if header != "" { - r.Header.Set(headerKeyAgentStats, header) - log.Printf("D! stats header (%s/%s): %s", operation, request, header) - } else { - log.Printf("W! empty stats header (%s/%s)", operation, request) - } } } From 82e2f395a8cb7f3847e0e9302f39a2ca63fac556 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 27 Oct 2023 15:30:13 -0400 Subject: [PATCH 33/38] Fix translator unit tests --- .../appsignals_and_kubernetes_config.yaml | 6 ++++++ .../sampleConfig/base_appsignals_config.yaml | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml index 95d3f4bdf6..87d088acbd 100644 --- a/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/appsignals_and_kubernetes_config.yaml @@ -286,6 +286,11 @@ extensions: stats: operations: - PutLogEvents + agenthealth/traces: + is_usage_data_enabled: true + stats: + operations: + - PutTraceSegments processors: awsappsignals: resolvers: @@ -611,6 +616,7 @@ service: extensions: - awsproxy/app_signals - agenthealth/logs + - agenthealth/traces pipelines: metrics/app_signals: exporters: diff --git a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml index 97db0160aa..0d0aead667 100644 --- a/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_appsignals_config.yaml @@ -121,6 +121,16 @@ extensions: proxy_address: "" region: "" role_arn: "" + agenthealth/logs: + is_usage_data_enabled: true + stats: + operations: + - PutLogEvents + agenthealth/traces: + is_usage_data_enabled: true + stats: + operations: + - PutTraceSegments processors: awsappsignals: resolvers: @@ -414,6 +424,8 @@ receivers: service: extensions: - awsproxy/app_signals + - agenthealth/traces + - agenthealth/logs pipelines: metrics/app_signals: exporters: From 4e607eded78186629d7b878c3841b5121df42645 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Fri, 27 Oct 2023 17:18:53 -0400 Subject: [PATCH 34/38] Fix refactoring issues --- .../internal/resolver/attributesresolver.go | 2 +- .../awsappsignals/internal/resolver/eks.go | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go b/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go index 454c0f679a..724c43f065 100644 --- a/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go +++ b/plugins/processors/awsappsignals/internal/resolver/attributesresolver.go @@ -77,7 +77,7 @@ func (h *hostedInAttributeResolver) Process(attributes, resourceAttributes pcomm } } - if _, ok := resourceAttributes.Get(attr.HostedInEnvironment); !ok { + if _, ok := resourceAttributes.Get(attr.AWSHostedInEnvironment); !ok { hostedInEnv := "Generic" attributes.PutStr(attr.HostedInEnvironment, hostedInEnv) } diff --git a/plugins/processors/awsappsignals/internal/resolver/eks.go b/plugins/processors/awsappsignals/internal/resolver/eks.go index af04b42ee1..f5e694b054 100644 --- a/plugins/processors/awsappsignals/internal/resolver/eks.go +++ b/plugins/processors/awsappsignals/internal/resolver/eks.go @@ -30,7 +30,7 @@ import ( const ( // kubeAllowedStringAlphaNums holds the characters allowed in replicaset names from as parent deployment - // https://github.com/kubernetes/apimachinery/blob/master/pkg/util/rand/rand.go#L83 + // https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/rand/rand.go#L121 kubeAllowedStringAlphaNums = "bcdfghjklmnpqrstvwxz2456789" // Deletion delay adjustment: @@ -46,13 +46,18 @@ const ( ) var DefaultHostedInAttributeMap = map[string]string{ - semconv.AttributeK8SNamespaceName: attr.HostedInClusterName, + semconv.AttributeK8SNamespaceName: attr.HostedInK8SNamespace, } var ( - // ReplicaSet name = Deployment name + "-" + 10 alphanumeric characters long string (see https://stackoverflow.com/questions/46204504/kubernetes-pod-naming-convention and https://stackoverflow.com/questions/71090356/how-does-random-string-in-kubernetes-pod-name-decided) - // the alphanumeric characters in Kubernetes are restricted to exclude vowels to reduce the chances of "normal words" being formed. (see https://github.com/kubernetes/apimachinery/blob/master/pkg/util/rand/rand.go#L83) - replicaSetWithDeploymentNamePattern = fmt.Sprintf(`^(.+)-[%s]{10}$`, kubeAllowedStringAlphaNums) + // ReplicaSet name = Deployment name + "-" + up to 10 alphanumeric characters string, if the ReplicaSet was created through a deployment + // The suffix string of the ReplicaSet name is an int32 number (0 to 4,294,967,295) that is cast to a string and then + // mapped to an alphanumeric value with only the following characters allowed: "bcdfghjklmnpqrstvwxz2456789". + // The suffix string length is therefore nondeterministic. The regex accepts a suffix of length 6-10 to account for + // ReplicaSets not managed by deployments that may have similar names. + // Suffix Generation: https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/controller_utils.go#L1201 + // Alphanumeric Mapping: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/rand/rand.go#L121) + replicaSetWithDeploymentNamePattern = fmt.Sprintf(`^(.+)-[%s]{6,10}$`, kubeAllowedStringAlphaNums) deploymentFromReplicaSetPattern = regexp.MustCompile(replicaSetWithDeploymentNamePattern) // if a pod is launched directly by a replicaSet (with a given name by users), its name has the following pattern: // Pod name = ReplicaSet name + 5 alphanumeric characters long string From 325ff6186e5ad92f7685c3761661b250c4ba0178 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Fri, 27 Oct 2023 16:03:45 -0700 Subject: [PATCH 35/38] code refactoring on metric customization component of appsignals --- .../customconfiguration/common.go | 50 +++++---- .../customconfiguration/common_test.go | 20 ++++ .../customconfiguration/dropper.go | 12 +-- .../customconfiguration/dropper_test.go | 19 ++-- .../customconfiguration/keeper.go | 12 +-- .../customconfiguration/keeper_test.go | 15 ++- .../customconfiguration/replacer.go | 6 +- .../customconfiguration/replacer_test.go | 8 +- plugins/processors/awsappsignals/processor.go | 100 +++++++++--------- .../processor/awsappsignals/translator.go | 9 +- 10 files changed, 136 insertions(+), 115 deletions(-) create mode 100644 plugins/processors/awsappsignals/customconfiguration/common_test.go diff --git a/plugins/processors/awsappsignals/customconfiguration/common.go b/plugins/processors/awsappsignals/customconfiguration/common.go index 3b426f80cc..31d1ea8c0e 100644 --- a/plugins/processors/awsappsignals/customconfiguration/common.go +++ b/plugins/processors/awsappsignals/customconfiguration/common.go @@ -4,12 +4,21 @@ package customconfiguration import ( + "errors" "fmt" "github.com/gobwas/glob" "go.opentelemetry.io/collector/pdata/pcommon" ) +type AllowListAction string + +const ( + AllowListActionKeep AllowListAction = "keep" + AllowListActionDrop AllowListAction = "drop" + AllowListActionReplace AllowListAction = "replace" +) + type Selector struct { Dimension string `mapstructure:"dimension"` Match string `mapstructure:"match"` @@ -21,10 +30,10 @@ type Replacement struct { } type Rule struct { - Selectors []Selector `mapstructure:"selectors"` - Replacements []Replacement `mapstructure:"replacements,omitempty"` - Action string `mapstructure:"action"` - RuleName string `mapstructure:"rule_name,omitempty"` + Selectors []Selector `mapstructure:"selectors"` + Replacements []Replacement `mapstructure:"replacements,omitempty"` + Action AllowListAction `mapstructure:"action"` + RuleName string `mapstructure:"rule_name,omitempty"` } type SelectorMatcherItem struct { @@ -44,6 +53,18 @@ var traceKeyMap = map[string]string{ "RemoteOperation": "aws.remote.operation", } +func GetAllowListAction(action string) (AllowListAction, error) { + switch action { + case "drop": + return AllowListActionDrop, nil + case "keep": + return AllowListActionKeep, nil + case "replace": + return AllowListActionReplace, nil + } + return "", errors.New("invalid action in rule") +} + func getExactKey(metricDimensionKey string, isTrace bool) string { if !isTrace { return metricDimensionKey @@ -56,7 +77,7 @@ func getExactKey(metricDimensionKey string, isTrace bool) string { return traceDimensionKey } -func isSelected(attributes pcommon.Map, selectorMatchers []SelectorMatcherItem, isTrace bool) (bool, error) { +func matchesSelectors(attributes pcommon.Map, selectorMatchers []SelectorMatcherItem, isTrace bool) (bool, error) { for _, item := range selectorMatchers { exactKey := getExactKey(item.Key, isTrace) value, ok := attributes.Get(exactKey) @@ -82,24 +103,7 @@ func generateSelectorMatchers(selectors []Selector) []SelectorMatcherItem { return selectorMatchers } -func generateTestAttributes(service string, operation string, remoteService string, remoteOperation string, - isTrace bool) pcommon.Map { - attributes := pcommon.NewMap() - if isTrace { - attributes.PutStr("aws.local.service", service) - attributes.PutStr("aws.local.operation", operation) - attributes.PutStr("aws.remote.service", remoteService) - attributes.PutStr("aws.remote.operation", remoteOperation) - } else { - attributes.PutStr("Service", service) - attributes.PutStr("Operation", operation) - attributes.PutStr("RemoteService", remoteService) - attributes.PutStr("RemoteOperation", remoteOperation) - } - return attributes -} - -func generateActionDetails(rules []Rule, action string) []ActionItem { +func generateActionDetails(rules []Rule, action AllowListAction) []ActionItem { var actionItems []ActionItem for _, rule := range rules { if rule.Action == action { diff --git a/plugins/processors/awsappsignals/customconfiguration/common_test.go b/plugins/processors/awsappsignals/customconfiguration/common_test.go new file mode 100644 index 0000000000..eb9e6d8eb2 --- /dev/null +++ b/plugins/processors/awsappsignals/customconfiguration/common_test.go @@ -0,0 +1,20 @@ +package customconfiguration + +import "go.opentelemetry.io/collector/pdata/pcommon" + +func generateTestAttributes(service string, operation string, remoteService string, remoteOperation string, + isTrace bool) pcommon.Map { + attributes := pcommon.NewMap() + if isTrace { + attributes.PutStr("aws.local.service", service) + attributes.PutStr("aws.local.operation", operation) + attributes.PutStr("aws.remote.service", remoteService) + attributes.PutStr("aws.remote.operation", remoteOperation) + } else { + attributes.PutStr("Service", service) + attributes.PutStr("Operation", operation) + attributes.PutStr("RemoteService", remoteService) + attributes.PutStr("RemoteOperation", remoteOperation) + } + return attributes +} diff --git a/plugins/processors/awsappsignals/customconfiguration/dropper.go b/plugins/processors/awsappsignals/customconfiguration/dropper.go index 35956c637a..8a3d9df971 100644 --- a/plugins/processors/awsappsignals/customconfiguration/dropper.go +++ b/plugins/processors/awsappsignals/customconfiguration/dropper.go @@ -9,25 +9,25 @@ type DropActions struct { Actions []ActionItem } -func NewCustomDropper(rules []Rule) *DropActions { +func NewDropper(rules []Rule) *DropActions { return &DropActions{ - Actions: generateActionDetails(rules, "drop"), + Actions: generateActionDetails(rules, AllowListActionDrop), } } -func (d *DropActions) ShouldBeDropped(attributes, _ pcommon.Map) (bool, error) { +func (d *DropActions) ShouldBeDropped(attributes pcommon.Map) (bool, error) { // nothing will be dropped if no rule is defined if d.Actions == nil || len(d.Actions) == 0 { return false, nil } for _, element := range d.Actions { - isMatched, err := isSelected(attributes, element.SelectorMatchers, false) + isMatched, err := matchesSelectors(attributes, element.SelectorMatchers, false) if isMatched { - // The datapoint will be dropped if one of keep rule matched + // drop the datapoint as one of drop rules is matched return true, nil } if err != nil { - // The datapoint will be kept if error is occurred in match process + // keep the datapoint as an error occurred in match process return false, err } } diff --git a/plugins/processors/awsappsignals/customconfiguration/dropper_test.go b/plugins/processors/awsappsignals/customconfiguration/dropper_test.go index 0b1be34859..3f89810eec 100644 --- a/plugins/processors/awsappsignals/customconfiguration/dropper_test.go +++ b/plugins/processors/awsappsignals/customconfiguration/dropper_test.go @@ -82,8 +82,7 @@ func TestDropperProcessor(t *testing.T) { }, } - testDropper := NewCustomDropper(config) - testMapPlaceHolder := pcommon.NewMap() + testDropper := NewDropper(config) isTrace := false testCases := []TestCaseForDropper{ @@ -107,7 +106,7 @@ func TestDropperProcessor(t *testing.T) { for i := range testCases { tt := testCases[i] t.Run(tt.name, func(t *testing.T) { - result, err := testDropper.ShouldBeDropped(tt.input, testMapPlaceHolder) + result, err := testDropper.ShouldBeDropped(tt.input) assert.NoError(t, err) assert.Equal(t, tt.output, result) }) @@ -115,9 +114,7 @@ func TestDropperProcessor(t *testing.T) { } func TestDropperProcessorWithNilConfig(t *testing.T) { - - testDropper := NewCustomDropper(nil) - testMapPlaceHolder := pcommon.NewMap() + testDropper := NewDropper(nil) isTrace := false testCases := []TestCaseForDropper{ @@ -141,7 +138,7 @@ func TestDropperProcessorWithNilConfig(t *testing.T) { for i := range testCases { tt := testCases[i] t.Run(tt.name, func(t *testing.T) { - result, err := testDropper.ShouldBeDropped(tt.input, testMapPlaceHolder) + result, err := testDropper.ShouldBeDropped(tt.input) assert.NoError(t, err) assert.Equal(t, tt.output, result) }) @@ -149,11 +146,9 @@ func TestDropperProcessorWithNilConfig(t *testing.T) { } func TestDropperProcessorWithEmptyConfig(t *testing.T) { + var config []Rule - config := []Rule{} - - testDropper := NewCustomDropper(config) - testMapPlaceHolder := pcommon.NewMap() + testDropper := NewDropper(config) isTrace := false testCases := []TestCaseForDropper{ @@ -177,7 +172,7 @@ func TestDropperProcessorWithEmptyConfig(t *testing.T) { for i := range testCases { tt := testCases[i] t.Run(tt.name, func(t *testing.T) { - result, err := testDropper.ShouldBeDropped(tt.input, testMapPlaceHolder) + result, err := testDropper.ShouldBeDropped(tt.input) assert.NoError(t, err) assert.Equal(t, tt.output, result) }) diff --git a/plugins/processors/awsappsignals/customconfiguration/keeper.go b/plugins/processors/awsappsignals/customconfiguration/keeper.go index 33a8eb9261..d401578896 100644 --- a/plugins/processors/awsappsignals/customconfiguration/keeper.go +++ b/plugins/processors/awsappsignals/customconfiguration/keeper.go @@ -9,25 +9,25 @@ type KeepActions struct { Actions []ActionItem } -func NewCustomKeeper(rules []Rule) *KeepActions { +func NewKeeper(rules []Rule) *KeepActions { return &KeepActions{ - Actions: generateActionDetails(rules, "keep"), + Actions: generateActionDetails(rules, AllowListActionKeep), } } -func (k *KeepActions) ShouldBeDropped(attributes, _ pcommon.Map) (bool, error) { +func (k *KeepActions) ShouldBeDropped(attributes pcommon.Map) (bool, error) { // nothing will be dropped if no keep rule is defined if k.Actions == nil || len(k.Actions) == 0 { return false, nil } for _, element := range k.Actions { - isMatched, err := isSelected(attributes, element.SelectorMatchers, false) + isMatched, err := matchesSelectors(attributes, element.SelectorMatchers, false) if isMatched { - // The data point will not be dropped if one of keep rule matched + // keep the datapoint as one of the keep rules is matched return false, nil } if err != nil { - // The data point will be dropped when error is found + // drop the datapoint as an error occurred in match process return true, err } } diff --git a/plugins/processors/awsappsignals/customconfiguration/keeper_test.go b/plugins/processors/awsappsignals/customconfiguration/keeper_test.go index 9d1a7a478c..818ace8056 100644 --- a/plugins/processors/awsappsignals/customconfiguration/keeper_test.go +++ b/plugins/processors/awsappsignals/customconfiguration/keeper_test.go @@ -69,8 +69,7 @@ func TestKeeperProcessor(t *testing.T) { }, } - testKeeper := NewCustomKeeper(config) - testMapPlaceHolder := pcommon.NewMap() + testKeeper := NewKeeper(config) isTrace := false testCases := []TestCaseForKeeper{ @@ -93,7 +92,7 @@ func TestKeeperProcessor(t *testing.T) { for i := range testCases { tt := testCases[i] t.Run(tt.name, func(t *testing.T) { - result, err := testKeeper.ShouldBeDropped(tt.input, testMapPlaceHolder) + result, err := testKeeper.ShouldBeDropped(tt.input) assert.NoError(t, err) assert.Equal(t, tt.output, result) }) @@ -101,8 +100,7 @@ func TestKeeperProcessor(t *testing.T) { } func TestKeeperProcessorWithNilConfig(t *testing.T) { - testKeeper := NewCustomKeeper(nil) - testMapPlaceHolder := pcommon.NewMap() + testKeeper := NewKeeper(nil) isTrace := false testCases := []TestCaseForKeeper{ @@ -130,7 +128,7 @@ func TestKeeperProcessorWithNilConfig(t *testing.T) { for i := range testCases { tt := testCases[i] t.Run(tt.name, func(t *testing.T) { - result, err := testKeeper.ShouldBeDropped(tt.input, testMapPlaceHolder) + result, err := testKeeper.ShouldBeDropped(tt.input) assert.NoError(t, err) assert.Equal(t, tt.output, result) }) @@ -141,8 +139,7 @@ func TestKeeperProcessorWithEmptyConfig(t *testing.T) { config := []Rule{} - testKeeper := NewCustomKeeper(config) - testMapPlaceHolder := pcommon.NewMap() + testKeeper := NewKeeper(config) isTrace := false testCases := []TestCaseForKeeper{ @@ -170,7 +167,7 @@ func TestKeeperProcessorWithEmptyConfig(t *testing.T) { for i := range testCases { tt := testCases[i] t.Run(tt.name, func(t *testing.T) { - result, err := testKeeper.ShouldBeDropped(tt.input, testMapPlaceHolder) + result, err := testKeeper.ShouldBeDropped(tt.input) assert.NoError(t, err) assert.Equal(t, tt.output, result) }) diff --git a/plugins/processors/awsappsignals/customconfiguration/replacer.go b/plugins/processors/awsappsignals/customconfiguration/replacer.go index 34555bb214..5ca120e7da 100644 --- a/plugins/processors/awsappsignals/customconfiguration/replacer.go +++ b/plugins/processors/awsappsignals/customconfiguration/replacer.go @@ -11,9 +11,9 @@ type ReplaceActions struct { Actions []ActionItem } -func NewCustomReplacer(rules []Rule) *ReplaceActions { +func NewReplacer(rules []Rule) *ReplaceActions { return &ReplaceActions{ - generateActionDetails(rules, "replace"), + generateActionDetails(rules, AllowListActionReplace), } } @@ -27,7 +27,7 @@ func (r *ReplaceActions) Process(attributes, _ pcommon.Map, isTrace bool) error finalRules := make(map[string]string) for i := len(actions) - 1; i >= 0; i = i - 1 { element := actions[i] - isMatched, _ := isSelected(attributes, element.SelectorMatchers, isTrace) + isMatched, _ := matchesSelectors(attributes, element.SelectorMatchers, isTrace) if !isMatched { continue } diff --git a/plugins/processors/awsappsignals/customconfiguration/replacer_test.go b/plugins/processors/awsappsignals/customconfiguration/replacer_test.go index 2d1e9ec751..e0c9f913fb 100644 --- a/plugins/processors/awsappsignals/customconfiguration/replacer_test.go +++ b/plugins/processors/awsappsignals/customconfiguration/replacer_test.go @@ -71,7 +71,7 @@ func TestReplacerProcess(t *testing.T) { }, } - testReplacer := NewCustomReplacer(config) + testReplacer := NewReplacer(config) testMapPlaceHolder := pcommon.NewMap() testCases := []TestCaseForReplacer{ @@ -168,7 +168,7 @@ func TestReplacerProcessWithPriority(t *testing.T) { }, } - testReplacer := NewCustomReplacer(config) + testReplacer := NewReplacer(config) testMapPlaceHolder := pcommon.NewMap() testCases := []TestCaseForReplacer{ @@ -216,7 +216,7 @@ func TestReplacerProcessWithPriority(t *testing.T) { func TestReplacerProcessWithNilConfig(t *testing.T) { - testReplacer := NewCustomReplacer(nil) + testReplacer := NewReplacer(nil) testMapPlaceHolder := pcommon.NewMap() testCases := []TestCaseForReplacer{ @@ -250,7 +250,7 @@ func TestReplacerProcessWithEmptyConfig(t *testing.T) { config := []Rule{} - testReplacer := NewCustomReplacer(config) + testReplacer := NewReplacer(config) testMapPlaceHolder := pcommon.NewMap() testCases := []TestCaseForReplacer{ diff --git a/plugins/processors/awsappsignals/processor.go b/plugins/processors/awsappsignals/processor.go index 1c9e63cf04..b70eb6a82b 100644 --- a/plugins/processors/awsappsignals/processor.go +++ b/plugins/processors/awsappsignals/processor.go @@ -27,8 +27,8 @@ type attributesMutator interface { Process(attributes, resourceAttributes pcommon.Map, isTrace bool) error } -type customAllowlistMutator interface { - ShouldBeDropped(attributes, resourceAttributes pcommon.Map) (bool, error) +type allowListMutator interface { + ShouldBeDropped(attributes pcommon.Map) (bool, error) } type stopper interface { @@ -36,13 +36,13 @@ type stopper interface { } type awsappsignalsprocessor struct { - logger *zap.Logger - config *Config - customReplacer *customconfiguration.ReplaceActions - customAllowlistMutators []customAllowlistMutator - metricMutators []attributesMutator - traceMutators []attributesMutator - stoppers []stopper + logger *zap.Logger + config *Config + replaceActions *customconfiguration.ReplaceActions + allowlistMutators []allowListMutator + metricMutators []attributesMutator + traceMutators []attributesMutator + stoppers []stopper } func (ap *awsappsignalsprocessor) Start(_ context.Context, _ component.Host) error { @@ -53,14 +53,14 @@ func (ap *awsappsignalsprocessor) Start(_ context.Context, _ component.Host) err attributesNormalizer := normalizer.NewAttributesNormalizer(ap.logger) ap.metricMutators = []attributesMutator{attributesResolver, attributesNormalizer} - ap.customReplacer = customconfiguration.NewCustomReplacer(ap.config.Rules) - ap.traceMutators = []attributesMutator{attributesResolver, attributesNormalizer, ap.customReplacer} + ap.replaceActions = customconfiguration.NewReplacer(ap.config.Rules) + ap.traceMutators = []attributesMutator{attributesResolver, attributesNormalizer, ap.replaceActions} - customKeeper := customconfiguration.NewCustomKeeper(ap.config.Rules) - ap.customAllowlistMutators = []customAllowlistMutator{customKeeper} + keeper := customconfiguration.NewKeeper(ap.config.Rules) + ap.allowlistMutators = []allowListMutator{keeper} - customDropper := customconfiguration.NewCustomDropper(ap.config.Rules) - ap.customAllowlistMutators = []customAllowlistMutator{customDropper} + dropper := customconfiguration.NewDropper(ap.config.Rules) + ap.allowlistMutators = []allowListMutator{dropper} return nil } @@ -126,27 +126,27 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(ctx context.Context, m case pmetric.MetricTypeGauge: dps := m.Gauge().DataPoints() for i := 0; i < dps.Len(); i++ { - for _, Mutator := range ap.metricMutators { - err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + for _, mutator := range ap.metricMutators { + err := mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } } } dps.RemoveIf(func(d pmetric.NumberDataPoint) bool { - for _, Mutator := range ap.customAllowlistMutators { - shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + for _, mutator := range ap.allowlistMutators { + shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) - return true - } else if shouldBeDropped { + } + if shouldBeDropped { return true } } return false }) for i := 0; i < dps.Len(); i++ { - err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + err := ap.replaceActions.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } @@ -154,27 +154,27 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(ctx context.Context, m case pmetric.MetricTypeSum: dps := m.Sum().DataPoints() for i := 0; i < dps.Len(); i++ { - for _, Mutator := range ap.metricMutators { - err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + for _, mutator := range ap.metricMutators { + err := mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } } } dps.RemoveIf(func(d pmetric.NumberDataPoint) bool { - for _, Mutator := range ap.customAllowlistMutators { - shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + for _, mutator := range ap.allowlistMutators { + shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) - return true - } else if shouldBeDropped { + } + if shouldBeDropped { return true } } return false }) for i := 0; i < dps.Len(); i++ { - err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + err := ap.replaceActions.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } @@ -182,27 +182,27 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(ctx context.Context, m case pmetric.MetricTypeHistogram: dps := m.Histogram().DataPoints() for i := 0; i < dps.Len(); i++ { - for _, Mutator := range ap.metricMutators { - err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + for _, mutator := range ap.metricMutators { + err := mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } } } dps.RemoveIf(func(d pmetric.HistogramDataPoint) bool { - for _, Mutator := range ap.customAllowlistMutators { - shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + for _, mutator := range ap.allowlistMutators { + shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) - return true - } else if shouldBeDropped { + } + if shouldBeDropped { return true } } return false }) for i := 0; i < dps.Len(); i++ { - err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + err := ap.replaceActions.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } @@ -210,27 +210,27 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(ctx context.Context, m case pmetric.MetricTypeExponentialHistogram: dps := m.ExponentialHistogram().DataPoints() for i := 0; i < dps.Len(); i++ { - for _, Mutator := range ap.metricMutators { - err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + for _, mutator := range ap.metricMutators { + err := mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } } } dps.RemoveIf(func(d pmetric.ExponentialHistogramDataPoint) bool { - for _, Mutator := range ap.customAllowlistMutators { - shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + for _, mutator := range ap.allowlistMutators { + shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) - return true - } else if shouldBeDropped { + } + if shouldBeDropped { return true } } return false }) for i := 0; i < dps.Len(); i++ { - err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + err := ap.replaceActions.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } @@ -238,27 +238,27 @@ func (ap *awsappsignalsprocessor) processMetricAttributes(ctx context.Context, m case pmetric.MetricTypeSummary: dps := m.Summary().DataPoints() for i := 0; i < dps.Len(); i++ { - for _, Mutator := range ap.metricMutators { - err := Mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) + for _, mutator := range ap.metricMutators { + err := mutator.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } } } dps.RemoveIf(func(d pmetric.SummaryDataPoint) bool { - for _, Mutator := range ap.customAllowlistMutators { - shouldBeDropped, err := Mutator.ShouldBeDropped(d.Attributes(), resourceAttribes) + for _, mutator := range ap.allowlistMutators { + shouldBeDropped, err := mutator.ShouldBeDropped(d.Attributes()) if err != nil { ap.logger.Debug(failedToProcessAttributeWithCustomRule, zap.Error(err)) - return true - } else if shouldBeDropped { + } + if shouldBeDropped { return true } } return false }) for i := 0; i < dps.Len(); i++ { - err := ap.customReplacer.Process(dps.At(i).Attributes(), resourceAttribes, false) + err := ap.replaceActions.Process(dps.At(i).Attributes(), resourceAttribes, false) if err != nil { ap.logger.Debug(failedToProcessAttribute, zap.Error(err)) } diff --git a/translator/translate/otel/processor/awsappsignals/translator.go b/translator/translate/otel/processor/awsappsignals/translator.go index fe197250a1..eb3f66192a 100644 --- a/translator/translate/otel/processor/awsappsignals/translator.go +++ b/translator/translate/otel/processor/awsappsignals/translator.go @@ -79,8 +79,13 @@ func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, if ruleName, ok := ruleMap["rule_name"]; ok { ruleConfig.RuleName = ruleName.(string) } - ruleConfig.Action = action - if action == "replace" { + + var err error + ruleConfig.Action, err = customconfiguration.GetAllowListAction(action) + if err != nil { + return nil, err + } + if ruleConfig.Action == customconfiguration.AllowListActionReplace { replacements, ok := ruleMap["replacements"] if !ok { return nil, errors.New("replace action set, but no replacements defined for service rule") From 5d8f7f8df084e19909d42366c2e8b178605abd85 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Sat, 28 Oct 2023 11:54:38 -0400 Subject: [PATCH 36/38] Rename customconfiguration to rules. Minor tweaks --- plugins/processors/awsappsignals/config.go | 6 +-- .../processors/awsappsignals/config_test.go | 12 ++--- .../awsappsignals/internal/resolver/eks.go | 52 ++----------------- plugins/processors/awsappsignals/processor.go | 10 ++-- .../{customconfiguration => rules}/common.go | 2 +- .../common_test.go | 5 +- .../{customconfiguration => rules}/dropper.go | 2 +- .../dropper_test.go | 2 +- .../{customconfiguration => rules}/keeper.go | 2 +- .../keeper_test.go | 2 +- .../replacer.go | 2 +- .../replacer_test.go | 2 +- .../processor/awsappsignals/translator.go | 22 ++++---- 13 files changed, 40 insertions(+), 81 deletions(-) rename plugins/processors/awsappsignals/{customconfiguration => rules}/common.go (99%) rename plugins/processors/awsappsignals/{customconfiguration => rules}/common_test.go (85%) rename plugins/processors/awsappsignals/{customconfiguration => rules}/dropper.go (96%) rename plugins/processors/awsappsignals/{customconfiguration => rules}/dropper_test.go (99%) rename plugins/processors/awsappsignals/{customconfiguration => rules}/keeper.go (96%) rename plugins/processors/awsappsignals/{customconfiguration => rules}/keeper_test.go (99%) rename plugins/processors/awsappsignals/{customconfiguration => rules}/replacer.go (98%) rename plugins/processors/awsappsignals/{customconfiguration => rules}/replacer_test.go (99%) diff --git a/plugins/processors/awsappsignals/config.go b/plugins/processors/awsappsignals/config.go index dfc01ec00a..82f7283c91 100644 --- a/plugins/processors/awsappsignals/config.go +++ b/plugins/processors/awsappsignals/config.go @@ -4,12 +4,12 @@ package awsappsignals import ( - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" ) type Config struct { - Resolvers []string `mapstructure:"resolvers"` - Rules []customconfiguration.Rule `mapstructure:"rules"` + Resolvers []string `mapstructure:"resolvers"` + Rules []rules.Rule `mapstructure:"rules"` } func (cfg *Config) Validate() error { diff --git a/plugins/processors/awsappsignals/config_test.go b/plugins/processors/awsappsignals/config_test.go index 02ff05f91d..4cb8f47df9 100644 --- a/plugins/processors/awsappsignals/config_test.go +++ b/plugins/processors/awsappsignals/config_test.go @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap/confmaptest" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" ) func TestLoadConfig(t *testing.T) { @@ -27,9 +27,9 @@ func TestLoadConfig(t *testing.T) { id: component.NewIDWithName("awsappsignals", ""), expected: &Config{ Resolvers: []string{"eks"}, - Rules: []customconfiguration.Rule{ + Rules: []rules.Rule{ { - Selectors: []customconfiguration.Selector{ + Selectors: []rules.Selector{ { Dimension: "Operation", Match: "* /api/visits/*", @@ -43,7 +43,7 @@ func TestLoadConfig(t *testing.T) { RuleName: "keep01", }, { - Selectors: []customconfiguration.Selector{ + Selectors: []rules.Selector{ { Dimension: "RemoteService", Match: "UnknownRemoteService", @@ -56,7 +56,7 @@ func TestLoadConfig(t *testing.T) { Action: "drop", }, { - Selectors: []customconfiguration.Selector{ + Selectors: []rules.Selector{ { Dimension: "Operation", Match: "* /api/visits/*", @@ -66,7 +66,7 @@ func TestLoadConfig(t *testing.T) { Match: "*", }, }, - Replacements: []customconfiguration.Replacement{ + Replacements: []rules.Replacement{ { TargetDimension: "RemoteOperation", Value: "ListPetsByCustomer", diff --git a/plugins/processors/awsappsignals/internal/resolver/eks.go b/plugins/processors/awsappsignals/internal/resolver/eks.go index f5e694b054..5c2dfb65b6 100644 --- a/plugins/processors/awsappsignals/internal/resolver/eks.go +++ b/plugins/processors/awsappsignals/internal/resolver/eks.go @@ -43,6 +43,8 @@ const ( // To mitigate this issue, we've introduced a 2-minute deletion delay. This ensures that any // metric data that arrives within those 2 minutes, containing the old IP, will still get mapped correctly to a service. deletionDelay = 2 * time.Minute + + jitterKubernetesAPISeconds = 10 ) var DefaultHostedInAttributeMap = map[string]string{ @@ -518,8 +520,8 @@ func getEksResolver(logger *zap.Logger) subResolver { logger.Fatal("Failed to create eks client", zap.Error(err)) } - // add a time jitter of 10 seconds - jitterSleep(10) + // jitter calls to the kubernetes api + jitterSleep(jitterKubernetesAPISeconds) sharedInformerFactory := informers.NewSharedInformerFactory(clientset, 0) podInformer := sharedInformerFactory.Core().V1().Pods().Informer() @@ -553,8 +555,6 @@ func getEksResolver(logger *zap.Logger) subResolver { workloadPodCount: podWatcher.workloadPodCount, safeStopCh: safeStopCh, } - - go instance.debugPrint() }) return instance @@ -587,50 +587,6 @@ func (e *eksResolver) GetWorkloadAndNamespaceByIP(ip string) (string, string, er return "", "", errors.New("no EKS workload found for ip: " + ip) } -func printSyncMap(name string, m *sync.Map, logger *zap.Logger) { - logger.Debug("", zap.String("MapName", name)) - m.Range(func(key, value interface{}) bool { - logger.Debug("", zap.Any("key", key), zap.Any("value", value)) - return true - }) - logger.Debug("DEBUG ====================") -} - -func (e *eksResolver) debugPrint() { - // call some logic every 5 minutes for ever - for { - select { - case <-time.After(5 * time.Minute): - e.debug() - case <-e.safeStopCh.ch: - return - } - } -} - -func (e *eksResolver) debug() { - e.logger.Debug("start debug print") - // print ipToServiceAndNamespace - printSyncMap("ipToServiceAndNamespace", e.ipToServiceAndNamespace, e.logger) - - // print serviceAndNamespaceToSelectors - printSyncMap("serviceAndNamespaceToSelectors", e.serviceAndNamespaceToSelectors, e.logger) - - // print ipToPod - printSyncMap("ipToPod", e.ipToPod, e.logger) - - // print podToWorkloadAndNamespace - printSyncMap("podToWorkloadAndNamespace", e.podToWorkloadAndNamespace, e.logger) - - // print workloadAndNamespaceToLabels - printSyncMap("workloadAndNamespaceToLabels", e.workloadAndNamespaceToLabels, e.logger) - - // print serviceToWorkload - e.logger.Debug("workload pod count", zap.Any("workloadPodCount", e.workloadPodCount)) - printSyncMap("serviceToWorkload", e.serviceToWorkload, e.logger) - e.logger.Debug("end debug print") -} - func (e *eksResolver) Process(attributes, resourceAttributes pcommon.Map) error { if value, ok := attributes.Get(attr.AWSRemoteService); ok { valueStr := value.AsString() diff --git a/plugins/processors/awsappsignals/processor.go b/plugins/processors/awsappsignals/processor.go index b70eb6a82b..4d5ca28e9a 100644 --- a/plugins/processors/awsappsignals/processor.go +++ b/plugins/processors/awsappsignals/processor.go @@ -12,9 +12,9 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/customconfiguration" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/normalizer" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/internal/resolver" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" ) const ( @@ -38,7 +38,7 @@ type stopper interface { type awsappsignalsprocessor struct { logger *zap.Logger config *Config - replaceActions *customconfiguration.ReplaceActions + replaceActions *rules.ReplaceActions allowlistMutators []allowListMutator metricMutators []attributesMutator traceMutators []attributesMutator @@ -53,13 +53,13 @@ func (ap *awsappsignalsprocessor) Start(_ context.Context, _ component.Host) err attributesNormalizer := normalizer.NewAttributesNormalizer(ap.logger) ap.metricMutators = []attributesMutator{attributesResolver, attributesNormalizer} - ap.replaceActions = customconfiguration.NewReplacer(ap.config.Rules) + ap.replaceActions = rules.NewReplacer(ap.config.Rules) ap.traceMutators = []attributesMutator{attributesResolver, attributesNormalizer, ap.replaceActions} - keeper := customconfiguration.NewKeeper(ap.config.Rules) + keeper := rules.NewKeeper(ap.config.Rules) ap.allowlistMutators = []allowListMutator{keeper} - dropper := customconfiguration.NewDropper(ap.config.Rules) + dropper := rules.NewDropper(ap.config.Rules) ap.allowlistMutators = []allowListMutator{dropper} return nil diff --git a/plugins/processors/awsappsignals/customconfiguration/common.go b/plugins/processors/awsappsignals/rules/common.go similarity index 99% rename from plugins/processors/awsappsignals/customconfiguration/common.go rename to plugins/processors/awsappsignals/rules/common.go index 31d1ea8c0e..3c8fd06058 100644 --- a/plugins/processors/awsappsignals/customconfiguration/common.go +++ b/plugins/processors/awsappsignals/rules/common.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package customconfiguration +package rules import ( "errors" diff --git a/plugins/processors/awsappsignals/customconfiguration/common_test.go b/plugins/processors/awsappsignals/rules/common_test.go similarity index 85% rename from plugins/processors/awsappsignals/customconfiguration/common_test.go rename to plugins/processors/awsappsignals/rules/common_test.go index eb9e6d8eb2..e2ab0fe9a1 100644 --- a/plugins/processors/awsappsignals/customconfiguration/common_test.go +++ b/plugins/processors/awsappsignals/rules/common_test.go @@ -1,4 +1,7 @@ -package customconfiguration +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package rules import "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/plugins/processors/awsappsignals/customconfiguration/dropper.go b/plugins/processors/awsappsignals/rules/dropper.go similarity index 96% rename from plugins/processors/awsappsignals/customconfiguration/dropper.go rename to plugins/processors/awsappsignals/rules/dropper.go index 8a3d9df971..38d3ed6cc5 100644 --- a/plugins/processors/awsappsignals/customconfiguration/dropper.go +++ b/plugins/processors/awsappsignals/rules/dropper.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package customconfiguration +package rules import "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/plugins/processors/awsappsignals/customconfiguration/dropper_test.go b/plugins/processors/awsappsignals/rules/dropper_test.go similarity index 99% rename from plugins/processors/awsappsignals/customconfiguration/dropper_test.go rename to plugins/processors/awsappsignals/rules/dropper_test.go index 3f89810eec..3ff91dea0f 100644 --- a/plugins/processors/awsappsignals/customconfiguration/dropper_test.go +++ b/plugins/processors/awsappsignals/rules/dropper_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package customconfiguration +package rules import ( "testing" diff --git a/plugins/processors/awsappsignals/customconfiguration/keeper.go b/plugins/processors/awsappsignals/rules/keeper.go similarity index 96% rename from plugins/processors/awsappsignals/customconfiguration/keeper.go rename to plugins/processors/awsappsignals/rules/keeper.go index d401578896..c88902a20a 100644 --- a/plugins/processors/awsappsignals/customconfiguration/keeper.go +++ b/plugins/processors/awsappsignals/rules/keeper.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package customconfiguration +package rules import "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/plugins/processors/awsappsignals/customconfiguration/keeper_test.go b/plugins/processors/awsappsignals/rules/keeper_test.go similarity index 99% rename from plugins/processors/awsappsignals/customconfiguration/keeper_test.go rename to plugins/processors/awsappsignals/rules/keeper_test.go index 818ace8056..8a8dbc4941 100644 --- a/plugins/processors/awsappsignals/customconfiguration/keeper_test.go +++ b/plugins/processors/awsappsignals/rules/keeper_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package customconfiguration +package rules import ( "testing" diff --git a/plugins/processors/awsappsignals/customconfiguration/replacer.go b/plugins/processors/awsappsignals/rules/replacer.go similarity index 98% rename from plugins/processors/awsappsignals/customconfiguration/replacer.go rename to plugins/processors/awsappsignals/rules/replacer.go index 5ca120e7da..7826581c68 100644 --- a/plugins/processors/awsappsignals/customconfiguration/replacer.go +++ b/plugins/processors/awsappsignals/rules/replacer.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package customconfiguration +package rules import ( "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/plugins/processors/awsappsignals/customconfiguration/replacer_test.go b/plugins/processors/awsappsignals/rules/replacer_test.go similarity index 99% rename from plugins/processors/awsappsignals/customconfiguration/replacer_test.go rename to plugins/processors/awsappsignals/rules/replacer_test.go index e0c9f913fb..7bd71c4211 100644 --- a/plugins/processors/awsappsignals/customconfiguration/replacer_test.go +++ b/plugins/processors/awsappsignals/rules/replacer_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package customconfiguration +package rules import ( "testing" diff --git a/translator/translate/otel/processor/awsappsignals/translator.go b/translator/translate/otel/processor/awsappsignals/translator.go index eb3f66192a..e8f2fcae64 100644 --- a/translator/translate/otel/processor/awsappsignals/translator.go +++ b/translator/translate/otel/processor/awsappsignals/translator.go @@ -12,7 +12,7 @@ import ( "go.opentelemetry.io/collector/processor" "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals" - "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/customconfiguration" + "github.com/aws/amazon-cloudwatch-agent/plugins/processors/awsappsignals/rules" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) @@ -66,11 +66,11 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, cfg *awsappsignals.Config) (component.Config, error) { - var rules []customconfiguration.Rule + var rules []rules.Rule rulesConfigKey := common.ConfigKey(configKey, common.AppSignalsRules) if conf.IsSet(rulesConfigKey) { for _, rule := range conf.Get(rulesConfigKey).([]interface{}) { - ruleConfig := customconfiguration.Rule{} + ruleConfig := rules.Rule{} ruleMap := rule.(map[string]interface{}) selectors := ruleMap["selectors"].([]interface{}) action := ruleMap["action"].(string) @@ -81,11 +81,11 @@ func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, } var err error - ruleConfig.Action, err = customconfiguration.GetAllowListAction(action) + ruleConfig.Action, err = rules.GetAllowListAction(action) if err != nil { return nil, err } - if ruleConfig.Action == customconfiguration.AllowListActionReplace { + if ruleConfig.Action == rules.AllowListActionReplace { replacements, ok := ruleMap["replacements"] if !ok { return nil, errors.New("replace action set, but no replacements defined for service rule") @@ -101,10 +101,10 @@ func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, return cfg, nil } -func getServiceSelectors(selectorsList []interface{}) []customconfiguration.Selector { - var selectors []customconfiguration.Selector +func getServiceSelectors(selectorsList []interface{}) []rules.Selector { + var selectors []rules.Selector for _, selector := range selectorsList { - selectorConfig := customconfiguration.Selector{} + selectorConfig := rules.Selector{} selectorsMap := selector.(map[string]interface{}) selectorConfig.Dimension = selectorsMap["dimension"].(string) @@ -114,10 +114,10 @@ func getServiceSelectors(selectorsList []interface{}) []customconfiguration.Sele return selectors } -func getServiceReplacements(replacementsList interface{}) []customconfiguration.Replacement { - var replacements []customconfiguration.Replacement +func getServiceReplacements(replacementsList interface{}) []rules.Replacement { + var replacements []rules.Replacement for _, replacement := range replacementsList.([]interface{}) { - replacementConfig := customconfiguration.Replacement{} + replacementConfig := rules.Replacement{} replacementMap := replacement.(map[string]interface{}) replacementConfig.TargetDimension = replacementMap["target_dimension"].(string) From 4eefb6bed76673853451107d438557aa66dc81a7 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Mon, 30 Oct 2023 09:59:59 -0400 Subject: [PATCH 37/38] Fix awsappsignals translation unit test --- .../translate/otel/processor/awsappsignals/translator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/translator/translate/otel/processor/awsappsignals/translator.go b/translator/translate/otel/processor/awsappsignals/translator.go index e8f2fcae64..de8c5f1d2a 100644 --- a/translator/translate/otel/processor/awsappsignals/translator.go +++ b/translator/translate/otel/processor/awsappsignals/translator.go @@ -66,7 +66,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, cfg *awsappsignals.Config) (component.Config, error) { - var rules []rules.Rule + var rulesList []rules.Rule rulesConfigKey := common.ConfigKey(configKey, common.AppSignalsRules) if conf.IsSet(rulesConfigKey) { for _, rule := range conf.Get(rulesConfigKey).([]interface{}) { @@ -93,9 +93,9 @@ func (t *translator) translateCustomRules(conf *confmap.Conf, configKey string, ruleConfig.Replacements = getServiceReplacements(replacements) } - rules = append(rules, ruleConfig) + rulesList = append(rulesList, ruleConfig) } - cfg.Rules = rules + cfg.Rules = rulesList } return cfg, nil From 908d99b069ae09e99689b651fc31378967eaf7b6 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Wed, 1 Nov 2023 11:07:30 -0400 Subject: [PATCH 38/38] Merge from main --- go.mod | 28 ++++++++++++++-------------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index e59ce1e3c8..be4dcfb8ca 100644 --- a/go.mod +++ b/go.mod @@ -6,38 +6,38 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2022 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231025143310-10cc00e9e0c4 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231027153828-85560f40a33d // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231027153828-85560f40a33d replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231025143310-10cc00e9e0c4 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231027153828-85560f40a33d -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231027153828-85560f40a33d replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713 replace go.opentelemetry.io/collector/config/confighttp => github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713 -replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231027153828-85560f40a33d // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 replace github.com/shirou/gopsutil/v3 => github.com/aws/telegraf/patches/gopsutil/v3 v3.0.0-20230915153624-7629361f8380 // indirect @@ -433,4 +433,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231023161526-9bd8785e9c2e +replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231027153828-85560f40a33d diff --git a/go.sum b/go.sum index 00c6bd03c6..ee73ceea57 100644 --- a/go.sum +++ b/go.sum @@ -146,38 +146,38 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713 h1:ACWoE8NqyI13oKdpg+074JIta5wmxsL1kOirTzDO5Gk= github.com/amazon-contributing/opentelemetry-collector-contrib/config/confighttp v0.0.0-20230928170322-0df38c533713/go.mod h1:4g9P7MZPReFnNJD0lpavI/LR0vIwlsTJov+hJoKT+nM= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231023161526-9bd8785e9c2e h1:7nfhePq8tBVC6edWESqcom0zIMBGsjl+lYm3cgnP6Kk= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:/8w8sPrpOeADRJgMsu8o4jOiFX29zCC899+ao7S1GXI= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231025143310-10cc00e9e0c4 h1:xX0G1Y0sAtsEcQoRpWn94gjTBPAAwzH4dk4U+CsssLY= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231025143310-10cc00e9e0c4/go.mod h1:UAXcRSojI8I0Kb9iS9a2v7J/iPrQ1loJIsBprSaVdFo= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231023161526-9bd8785e9c2e h1:wwkcWoKzZM1S92tSxSRxraQcUsJk+CS8UsKUR5Wcgow= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:cr4dmBlfnMVYT+gyKUAKh39zQu5u/UAukxQj15MdZ18= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231027153828-85560f40a33d h1:4aT8Ex9/LsNvp/J3FuElEzsWShjFW2GS6NT90z09WmU= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231027153828-85560f40a33d/go.mod h1:/8w8sPrpOeADRJgMsu8o4jOiFX29zCC899+ao7S1GXI= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231027153828-85560f40a33d h1:jCUfoiuYdDIh2rL3AXqnQIzuBfXGbIab0jCaUOsLOHM= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231027153828-85560f40a33d/go.mod h1:UAXcRSojI8I0Kb9iS9a2v7J/iPrQ1loJIsBprSaVdFo= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231027153828-85560f40a33d h1:DsayVZkW2sJSk8gOmdAz4uNtutHVbH1vTERMyaSBxZA= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231027153828-85560f40a33d/go.mod h1:cr4dmBlfnMVYT+gyKUAKh39zQu5u/UAukxQj15MdZ18= github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20231023230448-f645697bf350 h1:+75XAqf0Og8cshAdekRcqWf3v38Uw34XJRFbul6jbv0= github.com/amazon-contributing/opentelemetry-collector-contrib/extension/awsmiddleware v0.0.0-20231023230448-f645697bf350/go.mod h1:uOQa5/9Jle9VADEdWCXL4AbJr35NJQil30tapcTHQlw= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231023161526-9bd8785e9c2e h1:uQk5BvFVNMNCQswMx3gilWwPiqikvWx3BBFwMs85Stw= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231023161526-9bd8785e9c2e h1:FvMVzM0uAQmE1lPdKdmSCPpZnE0O3yg14J2oMwrBXt0= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231025143310-10cc00e9e0c4 h1:ADvOQZfcewhA9YZNLZFVYB0Wfi/19EsQtgA8RavELZk= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231025143310-10cc00e9e0c4/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231023161526-9bd8785e9c2e h1:CV/aQ8oNycuFDeMefLhWtobPuduwJjaC4uq6RsupGZM= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231023161526-9bd8785e9c2e h1:WhAzZRyuVJQG6mMCPPfGATY5O1rIxm2tFTQRRSo4fs4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231023161526-9bd8785e9c2e h1:RdPeBCsrUnkcm10fV2GbbT0mn3Y30mujrj20t5c7e10= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231027153828-85560f40a33d h1:mD+Yowd83cgOi5JvLLlkqdtM1OjVrh5h1tazO8UUM/Y= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231027153828-85560f40a33d/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231027153828-85560f40a33d h1:xyceDNef/z4LrNT00wGnCUdaymVqdnDBXUAv46M2ahs= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231027153828-85560f40a33d/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231027153828-85560f40a33d h1:FtIbMifCcVpuehF3eWDN8SSScDe5uSezQJNJPctpEWw= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231027153828-85560f40a33d/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231027153828-85560f40a33d h1:Qwha5VgT7YqcA4bqiT2iUNoOu/eIiU/UFf8QBCnvYY4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231027153828-85560f40a33d/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231027153828-85560f40a33d h1:cIlfirT7KzeUdRNOGrtd6MGHH6LxHSErJEfygsQr5+A= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231027153828-85560f40a33d/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231027153828-85560f40a33d h1:oIfODk8cssFqd7OcQdzGnBiOCmJQ3HwlNXhgF0sVY7E= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231027153828-85560f40a33d/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 h1:2daWNVtWNvRDoCTN5GG5N+LEM9OuY3RjJ0cboU3+xmM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231023161526-9bd8785e9c2e h1:p1xjUFPa5bBQrF42W8kKlb/KEc8EBP3QIsYmm6Uz3kw= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231027153828-85560f40a33d h1:9ggrXW27XY06+kColIRozGPdbjHlqWmWkT3S596E4BE= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231027153828-85560f40a33d/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713 h1:tKGat0aoXPkscaShYYRbnXH14asXqi1Iem4K3nMrNpk= github.com/amazon-contributing/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.0.0-20230928170322-0df38c533713/go.mod h1:fbCDqcaNUzfvbpI4y91hT8UfV18VyIKfS42BsPRDAuc= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231023161526-9bd8785e9c2e h1:X+WUoTqV/ya0a3i+BB5XjXfBcC+NMYaigOBf0tj44H8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231023161526-9bd8785e9c2e h1:EkTnV/B9ZKSp64Quycss0F+kYRY+7kc1M/nIWG43ubc= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231023161526-9bd8785e9c2e h1:ds+0GzYCxZGWDh56kg1XVUBg5o0bB6ES6T5jN/T+GAk= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231023161526-9bd8785e9c2e/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231027153828-85560f40a33d h1:uIDzHA/33UjBxk2WPC0Lo2UUiyahVnsraFz9cXithKI= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231027153828-85560f40a33d/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231027153828-85560f40a33d h1:rYgchItoanCKq7d6461905u3uASnmoKdezAnvvCJMgc= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231027153828-85560f40a33d/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231027153828-85560f40a33d h1:8g+GKEcezZDgnieQFnD5D+tUcC2N2Bc6uuXQaUG9OlI= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231027153828-85560f40a33d/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE=