[Bug Fix] Fix Excessive IMDS related error logging

extension/entitystore/ec2Info.go

@@ -32,7 +32,8 @@ type EC2Info struct {
 	AutoScalingGroup string
 
 	// region is used while making call to describeTags Ec2 API for AutoScalingGroup
-	Region string
+	Region         string
+	kubernetesMode string
 
 	metadataProvider ec2metadataprovider.MetadataProvider
 	logger           *zap.Logger
@@ -48,8 +49,11 @@ func (ei *EC2Info) initEc2Info() {
 	if err := ei.setInstanceIDAccountID(); err != nil {
 		return
 	}
-	if err := ei.setAutoScalingGroup(); err != nil {
-		return
+	// Instance metadata tags is not usable for EKS nodes
+	// https://github.com/kubernetes/cloud-provider-aws/issues/762
+	if ei.kubernetesMode == "" {
+		limitedRetryer := NewRetryer(true, true, defaultJitterMin, defaultJitterMax, ec2tagger.BackoffSleepArray, maxRetry, ei.done, ei.logger)
+		limitedRetryer.refreshLoop(ei.retrieveAsgName)
 	}
 	ei.logger.Debug("Finished initializing EC2Info")
 }
@@ -99,40 +103,6 @@ func (ei *EC2Info) setInstanceIDAccountID() error {
 	}
 }
 
-func (ei *EC2Info) setAutoScalingGroup() error {
-	retry := 0
-	for {
-		var waitDuration time.Duration
-		if retry < len(ec2tagger.BackoffSleepArray) {
-			waitDuration = ec2tagger.BackoffSleepArray[retry]
-		} else {
-			waitDuration = ec2tagger.BackoffSleepArray[len(ec2tagger.BackoffSleepArray)-1]
-		}
-
-		wait := time.NewTimer(waitDuration)
-		select {
-		case <-ei.done:
-			wait.Stop()
-			return errors.New("shutdown signal received")
-		case <-wait.C:
-		}
-
-		if retry > 0 {
-			ei.logger.Debug("Initial retrieval of tags and volumes", zap.Int("retry", retry))
-		}
-
-		if err := ei.retrieveAsgName(); err != nil {
-			ei.logger.Debug("Unable to fetch instance tags with imds", zap.Int("retry", retry), zap.Error(err))
-		} else {
-			ei.logger.Debug("Retrieval of auto-scaling group tags succeeded")
-			return nil
-		}
-
-		retry++
-	}
-
-}
-
 func (ei *EC2Info) retrieveAsgName() error {
 	tags, err := ei.metadataProvider.InstanceTags(context.Background())
 	if err != nil {
@@ -142,6 +112,7 @@ func (ei *EC2Info) retrieveAsgName() error {
 		asg, err := ei.metadataProvider.InstanceTagValue(context.Background(), ec2tagger.Ec2InstanceTagKeyASG)
 		if err != nil {
 			ei.logger.Error("Failed to get AutoScalingGroup through metadata provider", zap.Error(err))
+			return err
 		} else {
 			ei.logger.Debug("AutoScalingGroup retrieved through IMDS")
 			ei.mutex.Lock()
@@ -156,9 +127,10 @@ func (ei *EC2Info) retrieveAsgName() error {
 	return nil
 }
 
-func newEC2Info(metadataProvider ec2metadataprovider.MetadataProvider, done chan struct{}, region string, logger *zap.Logger) *EC2Info {
+func newEC2Info(metadataProvider ec2metadataprovider.MetadataProvider, kubernetesMode string, done chan struct{}, region string, logger *zap.Logger) *EC2Info {
 	return &EC2Info{
 		metadataProvider: metadataProvider,
+		kubernetesMode:   kubernetesMode,
 		done:             done,
 		Region:           region,
 		logger:           logger,

extension/entitystore/ec2Info_test.go

@@ -15,6 +15,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/aws/amazon-cloudwatch-agent/internal/ec2metadataprovider"
+	"github.com/aws/amazon-cloudwatch-agent/translator/config"
 )
 
 var mockedInstanceIdentityDoc = &ec2metadata.EC2InstanceIdentityDocument{
@@ -236,3 +237,37 @@ func TestNotInitIfMetadataProviderIsEmpty(t *testing.T) {
 		})
 	}
 }
+
+func TestNoASGRetrievalInKubernetesMode(t *testing.T) {
+	type args struct {
+		metadataProvider ec2metadataprovider.MetadataProvider
+		kubernetesMode   string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+		want    EC2Info
+	}{
+		{
+			name: "EKSNoASGFromEC2Info",
+			args: args{
+				metadataProvider: &mockMetadataProvider{InstanceIdentityDocument: mockedInstanceIdentityDoc, Tags: map[string]string{"aws:autoscaling:groupName": tagVal3}},
+				kubernetesMode:   config.ModeEKS,
+			},
+			wantErr: false,
+			want: EC2Info{
+				AutoScalingGroup: "",
+			},
+		},
+	}
+	for _, tt := range tests {
+		logger, _ := zap.NewDevelopment()
+		t.Run(tt.name, func(t *testing.T) {
+			ei := &EC2Info{metadataProvider: tt.args.metadataProvider, kubernetesMode: tt.args.kubernetesMode, logger: logger}
+			go ei.initEc2Info()
+			time.Sleep(3 * time.Second)
+			assert.Equal(t, tt.want.AutoScalingGroup, ei.GetAutoScalingGroup())
+		})
+	}
+}

extension/entitystore/extension.go

@@ -94,9 +94,13 @@ func (e *EntityStore) Start(ctx context.Context, host component.Host) error {
 	e.serviceprovider = newServiceProvider(e.mode, e.config.Region, &e.ec2Info, e.metadataprovider, getEC2Provider, ec2CredentialConfig, e.done, e.logger)
 	switch e.mode {
 	case config.ModeEC2:
-		e.ec2Info = *newEC2Info(e.metadataprovider, e.done, e.config.Region, e.logger)
+		e.ec2Info = *newEC2Info(e.metadataprovider, e.kubernetesMode, e.done, e.config.Region, e.logger)
 		go e.ec2Info.initEc2Info()
-		go e.serviceprovider.startServiceProvider()
+		// Instance metadata tags is not usable for EKS nodes
+		// https://github.com/kubernetes/cloud-provider-aws/issues/762
+		if e.kubernetesMode == "" {
+			go e.serviceprovider.startServiceProvider()
+		}
 	}
 	if e.kubernetesMode != "" {
 		e.eksInfo = newEKSInfo(e.logger)

extension/entitystore/extension_test.go

@@ -29,6 +29,7 @@ import (
 
 type mockServiceProvider struct {
 	mock.Mock
+	started bool
 }
 
 // This helper function creates a test logger
@@ -626,6 +627,59 @@ func TestEntityStore_LogMessageDoesNotIncludeResourceInfo(t *testing.T) {
 	}
 }
 
+func TestEntityStore_ServiceProviderInDifferentEnv(t *testing.T) {
+	type args struct {
+		metadataProvider ec2metadataprovider.MetadataProvider
+		mode             string
+		kubernetesMode   string
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "EC2inEKS",
+			args: args{
+				mode:           config.ModeEC2,
+				kubernetesMode: config.ModeEKS,
+			},
+		},
+		{
+			name: "EC2Only",
+			args: args{
+				mode: config.ModeEC2,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			esConfig := &Config{
+				Mode:           tt.args.mode,
+				KubernetesMode: tt.args.kubernetesMode,
+			}
+			getMetaDataProvider = mockMetadataProviderFunc
+			e := EntityStore{
+				logger: zap.NewNop(),
+				config: esConfig,
+			}
+			e.Start(nil, nil)
+			time.Sleep(3 * time.Second)
+
+			name, source := e.serviceprovider.getServiceNameAndSource()
+			if tt.args.mode == config.ModeEC2 && tt.args.kubernetesMode != "" {
+				assert.Equal(t, name, ServiceNameUnknown)
+				assert.Equal(t, source, ServiceNameSourceUnknown)
+			} else if tt.args.mode == config.ModeEC2 && tt.args.kubernetesMode == "" {
+				assert.Equal(t, name, "TestRole")
+				assert.Equal(t, source, ServiceNameSourceClientIamRole)
+			}
+
+		})
+	}
+
+}
+
 func assertIfNonEmpty(t *testing.T, message string, pattern string) {
 	if pattern != "" {
 		assert.NotContains(t, message, pattern)

internal/retryer/imdsretryer.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/influxdata/wlog"
 
 	"github.com/aws/amazon-cloudwatch-agent/cfg/envconfig"
 )
@@ -43,7 +44,7 @@ func (r IMDSRetryer) ShouldRetry(req *request.Request) bool {
 	if awsError, ok := req.Error.(awserr.Error); r.DefaultRetryer.ShouldRetry(req) || (ok && awsError != nil && awsError.Code() == "EC2MetadataError") {
 		shouldRetry = true
 	}
-	fmt.Printf("D! should retry %t for imds error : %v", shouldRetry, req.Error)
+	fmtDebugLog(wlog.LogLevel(), "D! should retry %t for imds error : %v", shouldRetry, req.Error)
 	return shouldRetry
 }
 
@@ -55,3 +56,10 @@ func GetDefaultRetryNumber() int {
 	}
 	return DefaultImdsRetries
 }
+
+// fmtDebugLog logs the content only if the log level is DEBUG
+func fmtDebugLog(logLevel wlog.Level, format string, args ...interface{}) {
+	if logLevel == wlog.DEBUG {
+		fmt.Printf(format, args...)
+	}
+}