From 64d9a5cff2c0a30936627d21503d40a81c82e5ba Mon Sep 17 00:00:00 2001 From: Seth L <81644108+sethAmazon@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:19:44 -0400 Subject: [PATCH] Add New Metrics For Where Customer Are Using The Agent (#913) --- cmd/config-downloader/downloader.go | 2 +- handlers/agentinfo/info.go | 41 ++++++++++++++++--- handlers/agentinfo/info_test.go | 16 +++++++- plugins/outputs/cloudwatch/cloudwatch.go | 2 +- plugins/outputs/cloudwatch/cloudwatch_test.go | 2 +- plugins/outputs/cloudwatch/config.go | 4 +- .../outputs/cloudwatchlogs/cloudwatchlogs.go | 6 ++- plugins/outputs/cloudwatchlogs/pusher_test.go | 2 +- translator/context/context.go | 10 +++++ .../sampleConfig/advanced_config_darwin.yaml | 2 + .../sampleConfig/advanced_config_linux.yaml | 2 + .../sampleConfig/advanced_config_windows.yaml | 2 + .../sampleConfig/basic_config_linux.yaml | 2 + .../sampleConfig/basic_config_windows.yaml | 2 + .../sampleConfig/collectd_config_linux.yaml | 2 + .../sampleConfig/complete_darwin_config.conf | 2 + .../sampleConfig/complete_darwin_config.yaml | 2 + .../sampleConfig/complete_linux_config.conf | 2 + .../sampleConfig/complete_linux_config.yaml | 2 + .../sampleConfig/complete_windows_config.conf | 2 + .../sampleConfig/complete_windows_config.yaml | 2 + .../sampleConfig/delta_config_linux.yaml | 2 + .../sampleConfig/delta_net_config_linux.yaml | 2 + .../sampleConfig/drop_origin_linux.yaml | 2 + .../ignore_append_dimensions.yaml | 2 + .../sampleConfig/invalid_input_linux.yaml | 2 + .../sampleConfig/standard_config_linux.yaml | 2 + ...ndard_config_linux_with_common_config.yaml | 2 + .../sampleConfig/standard_config_windows.yaml | 2 + ...ard_config_windows_with_common_config.yaml | 2 + .../sampleConfig/statsd_config_linux.yaml | 2 + .../sampleConfig/statsd_config_windows.yaml | 2 + translator/tocwconfig/tocwconfig_test.go | 13 +++++- translator/tocwconfig/tocwconfig_unix_test.go | 10 ++++- .../tocwconfig/tocwconfig_windows_test.go | 8 +++- translator/translate/agent/agent.go | 3 ++ translator/translate/agent/ruleRegion.go | 9 ++-- translator/translate/logs/logs_test.go | 16 ++++++++ .../translate/logs/ruleBasicLogConfig.go | 3 ++ .../otel/exporter/awscloudwatch/translator.go | 3 ++ translator/util/sdkutil.go | 9 +++- 41 files changed, 183 insertions(+), 22 deletions(-) diff --git a/cmd/config-downloader/downloader.go b/cmd/config-downloader/downloader.go index 477173104f..fc198b3db1 100644 --- a/cmd/config-downloader/downloader.go +++ b/cmd/config-downloader/downloader.go @@ -143,7 +143,7 @@ func main() { mode = sdkutil.DetectAgentMode(mode) - region = util.DetectRegion(mode, cc.CredentialsMap()) + region, _ = util.DetectRegion(mode, cc.CredentialsMap()) if region == "" && downloadLocation != locationDefault { fmt.Println("Unable to determine aws-region.") diff --git a/handlers/agentinfo/info.go b/handlers/agentinfo/info.go index 22c3868d2d..5e95aae31b 100644 --- a/handlers/agentinfo/info.go +++ b/handlers/agentinfo/info.go @@ -30,12 +30,22 @@ import ( "github.com/aws/amazon-cloudwatch-agent/cfg/envconfig" "github.com/aws/amazon-cloudwatch-agent/internal/util/collections" "github.com/aws/amazon-cloudwatch-agent/receiver/adapter" + translatorConfig "github.com/aws/amazon-cloudwatch-agent/translator/config" ) const ( versionFilename = "CWAGENT_VERSION" unknownVersion = "Unknown" updateInterval = time.Minute + // region types + AgentConfigJson = "ACJ" + CredsMap = "CM" + EC2Metadata = "EC2M" + ECSMetadata = "ECSM" + RegionNotFound = "RNF" + ModeEC2 = "EC2" + ModeOnPrem = "OP" + ModeWithIRSA = "WI" ) var ( @@ -50,10 +60,12 @@ var ( id = uuid.NewString() sharedConfigFallback atomic.Bool imdsFallbackSucceed atomic.Bool + isRunningAsRoot = defaultIsRunningAsRoot + runInContainer *int + regionType *string + mode *string ) -var isRunningAsRoot = defaultIsRunningAsRoot - type AgentInfo interface { RecordOpData(time.Duration, int, error) StatsHeader() string @@ -77,15 +89,21 @@ type agentStats struct { StatusCode *int `json:"code,omitempty"` SharedConfigFallback *int `json:"scfb,omitempty"` ImdsFallbackSucceed *int `json:"ifs,omitempty"` + RunInContainer *int `json:"ric,omitempty"` + RegionType *string `json:"rt,omitempty"` + Mode *string `json:"m,omitempty"` } -func New(groupName string) AgentInfo { - return newAgentInfo(groupName) +func New(groupName string, regionType string, mode string) AgentInfo { + return newAgentInfo(groupName, regionType, mode) } -func newAgentInfo(groupName string) *agentInfo { +func newAgentInfo(groupName string, regionTypeInput string, modeInput string) *agentInfo { ai := new(agentInfo) ai.userAgent = getUserAgent(groupName, fullVersion, receivers, processors, exporters, isUsageDataEnabled()) + runInContainer = runInContainerFunc() + regionType = aws.String(regionTypeInput) + mode = aws.String(modeInput) if isUsageDataEnabled() { ai.proc, _ = process.NewProcess(int32(os.Getpid())) if ai.proc == nil { @@ -96,6 +114,9 @@ func newAgentInfo(groupName string) *agentInfo { MemoryBytes: ai.memoryBytes(), FileDescriptorCount: ai.fileDescriptorCount(), ThreadCount: ai.threadCount(), + RunInContainer: runInContainer, + RegionType: regionType, + Mode: mode, } ai.statsHeader = getAgentStats(stats) ai.nextUpdate = time.Now().Add(updateInterval) @@ -126,6 +147,9 @@ func (ai *agentInfo) RecordOpData(latency time.Duration, payloadBytes int, err e stats.ThreadCount = ai.threadCount() stats.SharedConfigFallback = getSharedConfigFallback() stats.ImdsFallbackSucceed = succeedImdsFallback() + stats.RunInContainer = runInContainer + stats.RegionType = regionType + stats.Mode = mode ai.nextUpdate = now.Add(updateInterval) } @@ -330,3 +354,10 @@ func getSharedConfigFallback() *int { func SetImdsFallbackSucceed() { imdsFallbackSucceed.Store(true) } + +func runInContainerFunc() *int { + if os.Getenv(translatorConfig.RUN_IN_CONTAINER) == translatorConfig.RUN_IN_CONTAINER_TRUE { + return aws.Int(1) + } + return aws.Int(0) +} diff --git a/handlers/agentinfo/info_test.go b/handlers/agentinfo/info_test.go index 5a7550568d..edc0b05dd8 100644 --- a/handlers/agentinfo/info_test.go +++ b/handlers/agentinfo/info_test.go @@ -29,7 +29,7 @@ import ( ) func TestNew(t *testing.T) { - ai := New("") + ai := New("", "", "") expectedUserAgentRegex := `^CWAgent/Unknown \(.*\) ` + `ID/[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$` @@ -37,7 +37,7 @@ func TestNew(t *testing.T) { } func TestRecordOpData(t *testing.T) { - ai := newAgentInfo("") + ai := newAgentInfo("", "", "") stats := ai.StatsHeader() actual := agentStats{} @@ -124,8 +124,20 @@ func TestGetAgentStats(t *testing.T) { PayloadBytes: aws.Int(5678), StatusCode: aws.Int(200), ImdsFallbackSucceed: aws.Int(1), + RunInContainer: aws.Int(0), + RegionType: aws.String(EC2Metadata), + Mode: aws.String(ModeWithIRSA), } + assert.Equal(t, "\"cpu\":1.2,\"mem\":123,\"fd\":456,\"th\":789,\"lat\":1234,\"load\":5678,\"code\":200,\"ifs\":1,\"ric\":0,\"rt\":\"EC2M\",\"m\":\"WI\"", getAgentStats(stats)) + + stats.Mode = nil + assert.Equal(t, "\"cpu\":1.2,\"mem\":123,\"fd\":456,\"th\":789,\"lat\":1234,\"load\":5678,\"code\":200,\"ifs\":1,\"ric\":0,\"rt\":\"EC2M\"", getAgentStats(stats)) + + stats.RegionType = nil + assert.Equal(t, "\"cpu\":1.2,\"mem\":123,\"fd\":456,\"th\":789,\"lat\":1234,\"load\":5678,\"code\":200,\"ifs\":1,\"ric\":0", getAgentStats(stats)) + + stats.RunInContainer = nil assert.Equal(t, "\"cpu\":1.2,\"mem\":123,\"fd\":456,\"th\":789,\"lat\":1234,\"load\":5678,\"code\":200,\"ifs\":1", getAgentStats(stats)) stats.ImdsFallbackSucceed = nil diff --git a/plugins/outputs/cloudwatch/cloudwatch.go b/plugins/outputs/cloudwatch/cloudwatch.go index f4d14cdb74..69cb5b96c9 100644 --- a/plugins/outputs/cloudwatch/cloudwatch.go +++ b/plugins/outputs/cloudwatch/cloudwatch.go @@ -81,7 +81,7 @@ func (c *CloudWatch) Capabilities() consumer.Capabilities { } func (c *CloudWatch) Start(_ context.Context, host component.Host) error { - c.agentInfo = agentinfo.New("") + c.agentInfo = agentinfo.New("", c.config.RegionType, c.config.Mode) c.publisher, _ = publisher.NewPublisher( publisher.NewNonBlockingFifoQueue(metricChanBufferSize), maxConcurrentPublisher, diff --git a/plugins/outputs/cloudwatch/cloudwatch_test.go b/plugins/outputs/cloudwatch/cloudwatch_test.go index 7e74dc7fae..d95d68cef8 100644 --- a/plugins/outputs/cloudwatch/cloudwatch_test.go +++ b/plugins/outputs/cloudwatch/cloudwatch_test.go @@ -382,7 +382,7 @@ func newCloudWatchClient( MaxDatumsPerCall: defaultMaxDatumsPerCall, MaxValuesPerDatum: defaultMaxValuesPerDatum, }, - agentInfo: agentinfo.New(""), + agentInfo: agentinfo.New("", "", ""), } cloudwatch.startRoutines() return cloudwatch diff --git a/plugins/outputs/cloudwatch/config.go b/plugins/outputs/cloudwatch/config.go index 0790b54216..9c324529bb 100644 --- a/plugins/outputs/cloudwatch/config.go +++ b/plugins/outputs/cloudwatch/config.go @@ -11,13 +11,15 @@ import ( "go.opentelemetry.io/collector/component" ) -// Config represent a configuration for the CloudWatch logs exporter. +// Config represent a configuration for the CloudWatch metrics exporter. type Config struct { Region string `mapstructure:"region"` EndpointOverride string `mapstructure:"endpoint_override,omitempty"` AccessKey string `mapstructure:"access_key,omitempty"` SecretKey string `mapstructure:"secret_key,omitempty"` RoleARN string `mapstructure:"role_arn,omitempty"` + RegionType string `mapstructure:"region_type,omitempty"` + Mode string `mapstructure:"mode,omitempty"` Profile string `mapstructure:"profile,omitempty"` SharedCredentialFilename string `mapstructure:"shared_credential_file,omitempty"` Token string `mapstructure:"token,omitempty"` diff --git a/plugins/outputs/cloudwatchlogs/cloudwatchlogs.go b/plugins/outputs/cloudwatchlogs/cloudwatchlogs.go index 0cf6c93242..ba2dfc7ac3 100644 --- a/plugins/outputs/cloudwatchlogs/cloudwatchlogs.go +++ b/plugins/outputs/cloudwatchlogs/cloudwatchlogs.go @@ -31,7 +31,7 @@ const ( LogEntryField = "value" defaultFlushTimeout = 5 * time.Second - eventHeaderSize = 26 + eventHeaderSize = 200 truncatedSuffix = "[Truncated...]" msgSizeLimit = 256*1024 - eventHeaderSize @@ -43,6 +43,8 @@ const ( type CloudWatchLogs struct { Region string `toml:"region"` + RegionType string `toml:"region_type"` + Mode string `toml:"mode"` EndpointOverride string `toml:"endpoint_override"` AccessKey string `toml:"access_key"` SecretKey string `toml:"secret_key"` @@ -133,7 +135,7 @@ func (c *CloudWatchLogs) getDest(t Target) *cwDest { Logger: configaws.SDKLogger{}, }, ) - agentInfo := agentinfo.New(t.Group) + agentInfo := agentinfo.New(t.Group, c.RegionType, c.Mode) client.Handlers.Build.PushBackNamed(handlers.NewRequestCompressionHandler([]string{"PutLogEvents"})) client.Handlers.Build.PushBackNamed(handlers.NewCustomHeaderHandler("User-Agent", agentInfo.UserAgent())) client.Handlers.Build.PushBackNamed(handlers.NewDynamicCustomHeaderHandler("X-Amz-Agent-Stats", agentInfo.StatsHeader)) diff --git a/plugins/outputs/cloudwatchlogs/pusher_test.go b/plugins/outputs/cloudwatchlogs/pusher_test.go index 551f27e8ea..7695a98fbe 100644 --- a/plugins/outputs/cloudwatchlogs/pusher_test.go +++ b/plugins/outputs/cloudwatchlogs/pusher_test.go @@ -765,6 +765,6 @@ func TestResendWouldStopAfterExhaustedRetries(t *testing.T) { func testPreparation(retention int, s *svcMock, flushTimeout time.Duration, retryDuration time.Duration) (chan struct{}, *pusher) { stop := make(chan struct{}) - p := NewPusher(Target{"G", "S", retention}, s, flushTimeout, retryDuration, models.NewLogger("cloudwatchlogs", "test", ""), stop, &wg, agentinfo.New("")) + p := NewPusher(Target{"G", "S", retention}, s, flushTimeout, retryDuration, models.NewLogger("cloudwatchlogs", "test", ""), stop, &wg, agentinfo.New("", "", "")) return stop, p } diff --git a/translator/context/context.go b/translator/context/context.go index f4b88a4f4b..d39daff130 100644 --- a/translator/context/context.go +++ b/translator/context/context.go @@ -7,6 +7,7 @@ import ( "log" "os" + "github.com/aws/amazon-cloudwatch-agent/handlers/agentinfo" "github.com/aws/amazon-cloudwatch-agent/translator/config" ) @@ -41,6 +42,7 @@ type Context struct { multiConfig string outputTomlFilePath string mode string + shortMode string credentials map[string]string proxy map[string]string ssl map[string]string @@ -96,6 +98,10 @@ func (ctx *Context) Mode() string { return ctx.mode } +func (ctx *Context) ShortMode() string { + return ctx.shortMode +} + func (ctx *Context) Credentials() map[string]string { return ctx.credentials } @@ -112,12 +118,16 @@ func (ctx *Context) SetMode(mode string) { switch mode { case config.ModeEC2: ctx.mode = config.ModeEC2 + ctx.shortMode = agentinfo.ModeEC2 case config.ModeOnPrem: ctx.mode = config.ModeOnPrem + ctx.shortMode = agentinfo.ModeOnPrem case config.ModeOnPremise: ctx.mode = config.ModeOnPremise + ctx.shortMode = agentinfo.ModeOnPrem case config.ModeWithIRSA: ctx.mode = config.ModeWithIRSA + ctx.shortMode = agentinfo.ModeWithIRSA default: log.Panicf("Invalid mode %s. Valid mode values are %s, %s, %s and %s.", mode, config.ModeEC2, config.ModeOnPrem, config.ModeOnPremise, config.ModeWithIRSA) } diff --git a/translator/tocwconfig/sampleConfig/advanced_config_darwin.yaml b/translator/tocwconfig/sampleConfig/advanced_config_darwin.yaml index bb7860feb2..ad3d62c183 100644 --- a/translator/tocwconfig/sampleConfig/advanced_config_darwin.yaml +++ b/translator/tocwconfig/sampleConfig/advanced_config_darwin.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/advanced_config_linux.yaml b/translator/tocwconfig/sampleConfig/advanced_config_linux.yaml index c67bce7610..27d18cc365 100644 --- a/translator/tocwconfig/sampleConfig/advanced_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/advanced_config_linux.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/advanced_config_windows.yaml b/translator/tocwconfig/sampleConfig/advanced_config_windows.yaml index 2226ca01c1..76eea9ad06 100644 --- a/translator/tocwconfig/sampleConfig/advanced_config_windows.yaml +++ b/translator/tocwconfig/sampleConfig/advanced_config_windows.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/basic_config_linux.yaml b/translator/tocwconfig/sampleConfig/basic_config_linux.yaml index 676f41c6ad..79a0a303b6 100644 --- a/translator/tocwconfig/sampleConfig/basic_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/basic_config_linux.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-east-1 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/basic_config_windows.yaml b/translator/tocwconfig/sampleConfig/basic_config_windows.yaml index 851790b6e4..0406f716c4 100644 --- a/translator/tocwconfig/sampleConfig/basic_config_windows.yaml +++ b/translator/tocwconfig/sampleConfig/basic_config_windows.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/collectd_config_linux.yaml b/translator/tocwconfig/sampleConfig/collectd_config_linux.yaml index 420f130b3a..e656e8c192 100644 --- a/translator/tocwconfig/sampleConfig/collectd_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/collectd_config_linux.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/complete_darwin_config.conf b/translator/tocwconfig/sampleConfig/complete_darwin_config.conf index f46174f758..dd69467e71 100644 --- a/translator/tocwconfig/sampleConfig/complete_darwin_config.conf +++ b/translator/tocwconfig/sampleConfig/complete_darwin_config.conf @@ -126,4 +126,6 @@ force_flush_interval = "60s" log_stream_name = "LOG_STREAM_NAME" region = "us-west-2" + region_type = "ACJ" + mode = "EC2" role_arn = "log_role_arn_value_test" diff --git a/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml b/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml index 5c6d95a9d9..356b08993f 100644 --- a/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml +++ b/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml @@ -7,6 +7,8 @@ exporters: max_values_per_datum: 5000 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true role_arn: metrics_role_arn_value_test diff --git a/translator/tocwconfig/sampleConfig/complete_linux_config.conf b/translator/tocwconfig/sampleConfig/complete_linux_config.conf index c5f2b4f9c3..8cc4d07b61 100644 --- a/translator/tocwconfig/sampleConfig/complete_linux_config.conf +++ b/translator/tocwconfig/sampleConfig/complete_linux_config.conf @@ -126,4 +126,6 @@ force_flush_interval = "60s" log_stream_name = "LOG_STREAM_NAME" region = "us-west-2" + region_type = "ACJ" + mode = "EC2" role_arn = "log_role_arn_value_test" diff --git a/translator/tocwconfig/sampleConfig/complete_linux_config.yaml b/translator/tocwconfig/sampleConfig/complete_linux_config.yaml index d55c341d43..e3b6266a3b 100644 --- a/translator/tocwconfig/sampleConfig/complete_linux_config.yaml +++ b/translator/tocwconfig/sampleConfig/complete_linux_config.yaml @@ -10,6 +10,8 @@ exporters: max_values_per_datum: 5000 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true role_arn: metrics_role_arn_value_test diff --git a/translator/tocwconfig/sampleConfig/complete_windows_config.conf b/translator/tocwconfig/sampleConfig/complete_windows_config.conf index 26c8cf5cfb..9d06e7e62b 100644 --- a/translator/tocwconfig/sampleConfig/complete_windows_config.conf +++ b/translator/tocwconfig/sampleConfig/complete_windows_config.conf @@ -144,4 +144,6 @@ force_flush_interval = "60s" log_stream_name = "LOG_STREAM_NAME" region = "us-west-2" + region_type = "ACJ" + mode = "EC2" role_arn = "log_role_arn_value_test" diff --git a/translator/tocwconfig/sampleConfig/complete_windows_config.yaml b/translator/tocwconfig/sampleConfig/complete_windows_config.yaml index 284b6113f2..e199411548 100644 --- a/translator/tocwconfig/sampleConfig/complete_windows_config.yaml +++ b/translator/tocwconfig/sampleConfig/complete_windows_config.yaml @@ -7,6 +7,8 @@ exporters: max_values_per_datum: 5000 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true role_arn: metrics_role_arn_value_test diff --git a/translator/tocwconfig/sampleConfig/delta_config_linux.yaml b/translator/tocwconfig/sampleConfig/delta_config_linux.yaml index 1025e3674a..b2a937a028 100644 --- a/translator/tocwconfig/sampleConfig/delta_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/delta_config_linux.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-east-1 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/delta_net_config_linux.yaml b/translator/tocwconfig/sampleConfig/delta_net_config_linux.yaml index f609509f15..2c5a29d8fc 100644 --- a/translator/tocwconfig/sampleConfig/delta_net_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/delta_net_config_linux.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-east-1 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/drop_origin_linux.yaml b/translator/tocwconfig/sampleConfig/drop_origin_linux.yaml index 1049f465b4..a3149050e3 100644 --- a/translator/tocwconfig/sampleConfig/drop_origin_linux.yaml +++ b/translator/tocwconfig/sampleConfig/drop_origin_linux.yaml @@ -11,6 +11,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/ignore_append_dimensions.yaml b/translator/tocwconfig/sampleConfig/ignore_append_dimensions.yaml index 952734d325..839d3aee22 100644 --- a/translator/tocwconfig/sampleConfig/ignore_append_dimensions.yaml +++ b/translator/tocwconfig/sampleConfig/ignore_append_dimensions.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-east-1 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/invalid_input_linux.yaml b/translator/tocwconfig/sampleConfig/invalid_input_linux.yaml index caa37a5522..d6f929ce8c 100644 --- a/translator/tocwconfig/sampleConfig/invalid_input_linux.yaml +++ b/translator/tocwconfig/sampleConfig/invalid_input_linux.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-east-1 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/standard_config_linux.yaml b/translator/tocwconfig/sampleConfig/standard_config_linux.yaml index 28b536860d..a1a564ccd0 100644 --- a/translator/tocwconfig/sampleConfig/standard_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/standard_config_linux.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/standard_config_linux_with_common_config.yaml b/translator/tocwconfig/sampleConfig/standard_config_linux_with_common_config.yaml index 4d5f36b549..7b579184dc 100644 --- a/translator/tocwconfig/sampleConfig/standard_config_linux_with_common_config.yaml +++ b/translator/tocwconfig/sampleConfig/standard_config_linux_with_common_config.yaml @@ -7,6 +7,8 @@ exporters: namespace: CWAgent profile: AmazonCloudWatchAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true shared_credential_file: fake-path diff --git a/translator/tocwconfig/sampleConfig/standard_config_windows.yaml b/translator/tocwconfig/sampleConfig/standard_config_windows.yaml index ad93d6290d..d876d2434e 100644 --- a/translator/tocwconfig/sampleConfig/standard_config_windows.yaml +++ b/translator/tocwconfig/sampleConfig/standard_config_windows.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/standard_config_windows_with_common_config.yaml b/translator/tocwconfig/sampleConfig/standard_config_windows_with_common_config.yaml index 15e01ad980..336170336a 100644 --- a/translator/tocwconfig/sampleConfig/standard_config_windows_with_common_config.yaml +++ b/translator/tocwconfig/sampleConfig/standard_config_windows_with_common_config.yaml @@ -7,6 +7,8 @@ exporters: namespace: CWAgent profile: AmazonCloudWatchAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true shared_credential_file: fake-path diff --git a/translator/tocwconfig/sampleConfig/statsd_config_linux.yaml b/translator/tocwconfig/sampleConfig/statsd_config_linux.yaml index 8440d99174..63e1a71f32 100644 --- a/translator/tocwconfig/sampleConfig/statsd_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/statsd_config_linux.yaml @@ -7,6 +7,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/sampleConfig/statsd_config_windows.yaml b/translator/tocwconfig/sampleConfig/statsd_config_windows.yaml index a64e21b8a7..475f4c9f92 100644 --- a/translator/tocwconfig/sampleConfig/statsd_config_windows.yaml +++ b/translator/tocwconfig/sampleConfig/statsd_config_windows.yaml @@ -6,6 +6,8 @@ exporters: max_values_per_datum: 150 namespace: CWAgent region: us-west-2 + region_type: "ACJ" + mode: "EC2" resource_to_telemetry_conversion: enabled: true extensions: {} diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 816824aef8..aeeab33ce3 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -134,6 +134,7 @@ func TestStatsDConfig(t *testing.T) { for name, testCase := range testCases { t.Run(name, func(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) }) } @@ -142,6 +143,7 @@ func TestStatsDConfig(t *testing.T) { // Linux only for CollectD func TestCollectDConfig(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) expectedEnvVars := map[string]string{} checkTranslation(t, "collectd_config_linux", "linux", expectedEnvVars, "") checkTranslation(t, "collectd_config_linux", "darwin", nil, "") @@ -195,6 +197,7 @@ func TestBasicConfig(t *testing.T) { for name, testCase := range testCases { t.Run(name, func(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) }) } @@ -202,6 +205,7 @@ func TestBasicConfig(t *testing.T) { func TestInvalidInputConfig(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) expectedEnvVars := map[string]string{} checkTranslation(t, "invalid_input_linux", "linux", expectedEnvVars, "") } @@ -232,6 +236,7 @@ func TestStandardConfig(t *testing.T) { for name, testCase := range testCases { t.Run(name, func(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) t.Setenv(envconfig.IMDS_NUMBER_RETRY, "0") checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) }) @@ -262,6 +267,7 @@ func TestAdvancedConfig(t *testing.T) { for name, testCase := range testCases { t.Run(name, func(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) }) } @@ -414,6 +420,7 @@ func TestStandardConfigWithCommonConfig(t *testing.T) { for name, testCase := range testCases { t.Run(name, func(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) readCommonConfig(t, "./sampleConfig/commonConfig/withCredentialsProxySsl.toml") checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) }) @@ -422,6 +429,7 @@ func TestStandardConfigWithCommonConfig(t *testing.T) { func TestDeltaNetConfigLinux(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) expectedEnvVars := map[string]string{} checkTranslation(t, "delta_net_config_linux", "linux", expectedEnvVars, "") checkTranslation(t, "delta_net_config_linux", "darwin", nil, "") @@ -445,6 +453,7 @@ func TestLogFilterConfig(t *testing.T) { func TestIgnoreInvalidAppendDimensions(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) expectedEnvVars := map[string]string{} checkTranslation(t, "ignore_append_dimensions", "linux", expectedEnvVars, "") } @@ -499,8 +508,8 @@ func readCommonConfig(t *testing.T, commonConfigFilePath string) { func resetContext(t *testing.T) { t.Setenv(envconfig.IMDS_NUMBER_RETRY, strconv.Itoa(retryer.DefaultImdsRetries)) - util.DetectRegion = func(string, map[string]string) string { - return "us-west-2" + util.DetectRegion = func(string, map[string]string) (string, string) { + return "us-west-2", "ACJ" } util.DetectCredentialsPath = func() string { return "fake-path" diff --git a/translator/tocwconfig/tocwconfig_unix_test.go b/translator/tocwconfig/tocwconfig_unix_test.go index cd81848570..0d57e53b2d 100644 --- a/translator/tocwconfig/tocwconfig_unix_test.go +++ b/translator/tocwconfig/tocwconfig_unix_test.go @@ -6,10 +6,16 @@ package tocwconfig -import "testing" +import ( + "testing" + + "github.com/aws/amazon-cloudwatch-agent/translator/config" + "github.com/aws/amazon-cloudwatch-agent/translator/context" +) func TestCompleteConfigUnix(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) expectedEnvVars := map[string]string{ "CWAGENT_USER_AGENT": "CUSTOM USER AGENT VALUE", "CWAGENT_LOG_LEVEL": "DEBUG", @@ -24,6 +30,7 @@ func TestCompleteConfigUnix(t *testing.T) { func TestDeltaConfigLinux(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) expectedEnvVars := map[string]string{} checkTranslation(t, "delta_config_linux", "linux", expectedEnvVars, "") checkTranslation(t, "delta_config_linux", "darwin", nil, "") @@ -31,6 +38,7 @@ func TestDeltaConfigLinux(t *testing.T) { func TestDropOriginConfig(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) expectedEnvVars := map[string]string{} checkTranslation(t, "drop_origin_linux", "linux", expectedEnvVars, "") } diff --git a/translator/tocwconfig/tocwconfig_windows_test.go b/translator/tocwconfig/tocwconfig_windows_test.go index 7e074def52..166499a850 100644 --- a/translator/tocwconfig/tocwconfig_windows_test.go +++ b/translator/tocwconfig/tocwconfig_windows_test.go @@ -6,10 +6,16 @@ package tocwconfig -import "testing" +import ( + "testing" + + "github.com/aws/amazon-cloudwatch-agent/translator/config" + "github.com/aws/amazon-cloudwatch-agent/translator/context" +) func TestCompleteConfigWindows(t *testing.T) { resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) expectedEnvVars := map[string]string{ "CWAGENT_USER_AGENT": "CUSTOM USER AGENT VALUE", "CWAGENT_LOG_LEVEL": "DEBUG", diff --git a/translator/translate/agent/agent.go b/translator/translate/agent/agent.go index 2951f4ab81..dad8a19392 100644 --- a/translator/translate/agent/agent.go +++ b/translator/translate/agent/agent.go @@ -13,6 +13,7 @@ var ChildRule = map[string]translator.Rule{} const ( SectionKey = "agent" + Mode = "mode" ) func GetCurPath() string { @@ -27,6 +28,8 @@ type Agent struct { Interval string Credentials map[string]interface{} Region string + RegionType string + Mode string Internal bool Role_arn string } diff --git a/translator/translate/agent/ruleRegion.go b/translator/translate/agent/ruleRegion.go index aee69f4df8..dddfd32bba 100644 --- a/translator/translate/agent/ruleRegion.go +++ b/translator/translate/agent/ruleRegion.go @@ -6,6 +6,7 @@ package agent import ( "fmt" + "github.com/aws/amazon-cloudwatch-agent/handlers/agentinfo" "github.com/aws/amazon-cloudwatch-agent/translator" "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/util" @@ -15,20 +16,21 @@ type Region struct { } const ( - RegionKey = "region" + RegionKey = "region" + RegionType = "region_type" ) // This region will be provided to the corresponding input and output plugins // This should be applied before interpreting other component. func (r *Region) ApplyRule(input interface{}) (returnKey string, returnVal interface{}) { - var region string ctx := context.CurrentContext() _, inputRegion := translator.DefaultCase(RegionKey, "", input) if inputRegion != "" { Global_Config.Region = inputRegion.(string) + Global_Config.RegionType = agentinfo.AgentConfigJson return } - region = util.DetectRegion(ctx.Mode(), ctx.Credentials()) + region, regionType := util.DetectRegion(ctx.Mode(), ctx.Credentials()) if region == "" { translator.AddErrorMessages(GetCurPath()+"ruleRegion/", fmt.Sprintf("Region info is missing for mode: %s", @@ -36,6 +38,7 @@ func (r *Region) ApplyRule(input interface{}) (returnKey string, returnVal inter } Global_Config.Region = region + Global_Config.RegionType = regionType return } diff --git a/translator/translate/logs/logs_test.go b/translator/translate/logs/logs_test.go index e0bed36ef1..923386164f 100644 --- a/translator/translate/logs/logs_test.go +++ b/translator/translate/logs/logs_test.go @@ -19,6 +19,7 @@ import ( func TestLogs(t *testing.T) { l := new(Logs) agent.Global_Config.Region = "us-east-1" + agent.Global_Config.RegionType = "any" var input interface{} err := json.Unmarshal([]byte(`{"logs":{"log_stream_name":"LOG_STREAM_NAME"}}`), &input) @@ -32,6 +33,8 @@ func TestLogs(t *testing.T) { "cloudwatchlogs": []interface{}{ map[string]interface{}{ "region": "us-east-1", + "region_type": "any", + "mode": "", "log_stream_name": "LOG_STREAM_NAME", "force_flush_interval": "5s", }, @@ -44,6 +47,7 @@ func TestLogs(t *testing.T) { func TestLogs_LogStreamName(t *testing.T) { l := new(Logs) agent.Global_Config.Region = "us-east-1" + agent.Global_Config.RegionType = "any" var input interface{} err := json.Unmarshal([]byte(`{"logs":{}}`), &input) @@ -61,6 +65,8 @@ func TestLogs_LogStreamName(t *testing.T) { "cloudwatchlogs": []interface{}{ map[string]interface{}{ "region": "us-east-1", + "region_type": "any", + "mode": "OP", "log_stream_name": hostname, "force_flush_interval": "5s", }, @@ -86,6 +92,8 @@ func TestLogs_LogStreamName(t *testing.T) { "cloudwatchlogs": []interface{}{ map[string]interface{}{ "region": "us-east-1", + "region_type": "any", + "mode": "", "log_stream_name": "arn_aws_ecs_us-east-2_012345678910_task/cluster-name/9781c248-0edd-4cdb-9a93-f63cb662a5d3", "force_flush_interval": "5s", }, @@ -108,6 +116,8 @@ func TestLogs_LogStreamName(t *testing.T) { "cloudwatchlogs": []interface{}{ map[string]interface{}{ "region": "us-east-1", + "region_type": "any", + "mode": "", "log_stream_name": "demo-app-5ffc89b95c-jgnf6", "force_flush_interval": "5s", }, @@ -123,6 +133,7 @@ func TestLogs_LogStreamName(t *testing.T) { func TestLogs_ForceFlushInterval(t *testing.T) { l := new(Logs) agent.Global_Config.Region = "us-east-1" + agent.Global_Config.RegionType = "any" var input interface{} err := json.Unmarshal([]byte(`{"logs":{"force_flush_interval":10}}`), &input) @@ -140,6 +151,8 @@ func TestLogs_ForceFlushInterval(t *testing.T) { "cloudwatchlogs": []interface{}{ map[string]interface{}{ "region": "us-east-1", + "region_type": "any", + "mode": "OP", "log_stream_name": hostname, "force_flush_interval": "10s", }, @@ -155,6 +168,7 @@ func TestLogs_ForceFlushInterval(t *testing.T) { func TestLogs_EndpointOverride(t *testing.T) { l := new(Logs) agent.Global_Config.Region = "us-east-1" + agent.Global_Config.RegionType = "any" var input interface{} err := json.Unmarshal([]byte(`{"logs":{"endpoint_override":"https://logs-fips.us-east-1.amazonaws.com"}}`), &input) @@ -172,6 +186,8 @@ func TestLogs_EndpointOverride(t *testing.T) { "cloudwatchlogs": []interface{}{ map[string]interface{}{ "region": "us-east-1", + "region_type": "any", + "mode": "OP", "endpoint_override": "https://logs-fips.us-east-1.amazonaws.com", "log_stream_name": hostname, "force_flush_interval": "5s", diff --git a/translator/translate/logs/ruleBasicLogConfig.go b/translator/translate/logs/ruleBasicLogConfig.go index 07a9d37d37..28a4c9224a 100644 --- a/translator/translate/logs/ruleBasicLogConfig.go +++ b/translator/translate/logs/ruleBasicLogConfig.go @@ -5,6 +5,7 @@ package logs import ( "github.com/aws/amazon-cloudwatch-agent/translator" + "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" ) @@ -16,6 +17,8 @@ func (f *BasicLogConfig) ApplyRule(input interface{}) (returnKey string, returnV // add creds cloudwatchlogsConfig = translator.MergeTwoUniqueMaps(cloudwatchlogsConfig, agent.Global_Config.Credentials) cloudwatchlogsConfig[agent.RegionKey] = agent.Global_Config.Region + cloudwatchlogsConfig[agent.RegionType] = agent.Global_Config.RegionType + cloudwatchlogsConfig[agent.Mode] = context.CurrentContext().ShortMode() returnKey = Output_Cloudwatch_Logs returnVal = cloudwatchlogsConfig diff --git a/translator/translate/otel/exporter/awscloudwatch/translator.go b/translator/translate/otel/exporter/awscloudwatch/translator.go index da497d0a37..ea3ff13eb9 100644 --- a/translator/translate/otel/exporter/awscloudwatch/translator.go +++ b/translator/translate/otel/exporter/awscloudwatch/translator.go @@ -12,6 +12,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/internal/metric" "github.com/aws/amazon-cloudwatch-agent/plugins/outputs/cloudwatch" + "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/config" "github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/rollup_dimensions" @@ -57,6 +58,8 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { _ = credentials.Unmarshal(cfg) cfg.RoleARN = getRoleARN(conf) cfg.Region = agent.Global_Config.Region + cfg.RegionType = agent.Global_Config.RegionType + cfg.Mode = context.CurrentContext().ShortMode() if namespace, ok := common.GetString(conf, common.ConfigKey(common.MetricsKey, namespaceKey)); ok { cfg.Namespace = namespace } diff --git a/translator/util/sdkutil.go b/translator/util/sdkutil.go index 7d998e0c5b..de5d5a27f0 100644 --- a/translator/util/sdkutil.go +++ b/translator/util/sdkutil.go @@ -13,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/amazon-cloudwatch-agent/cfg/commonconfig" + "github.com/aws/amazon-cloudwatch-agent/handlers/agentinfo" "github.com/aws/amazon-cloudwatch-agent/translator" "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/util/ec2util" @@ -97,19 +98,25 @@ func defaultECSRegion() string { return ecsutil.GetECSUtilSingleton().Region } -func detectRegion(mode string, credsConfig map[string]string) (region string) { +func detectRegion(mode string, credsConfig map[string]string) (region string, regionType string) { region = SDKRegionWithCredsMap(mode, credsConfig) + regionType = agentinfo.RegionNotFound + if region != "" { + regionType = agentinfo.CredsMap + } // For ec2, fallback to metadata when no region info found in credential profile. if region == "" && mode == config.ModeEC2 { fmt.Println("I! Trying to detect region from ec2") region = DefaultEC2Region() + regionType = agentinfo.EC2Metadata } // try to get region from ecs metadata if region == "" && mode == config.ModeEC2 { fmt.Println("I! Trying to detect region from ecs") region = DefaultECSRegion() + regionType = agentinfo.ECSMetadata } return