[Application Signals] [TLS] Translate CWAgent Receiver JSON Config to…

… take in TLS settings (#1028)

translator/cmdutil/translatorutil.go

@@ -23,6 +23,7 @@ import (
 	"github.com/aws/amazon-cloudwatch-agent/translator/tocwconfig/toyamlconfig"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel"
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common"
 	translatorUtil "github.com/aws/amazon-cloudwatch-agent/translator/util"
 )
 
@@ -224,7 +225,31 @@ func TranslateJsonMapToYamlConfig(jsonConfigValue interface{}) (interface{}, err
 	if err = conf.Marshal(cfg); err != nil {
 		return nil, err
 	}
-	return conf.ToStringMap(), nil
+	strMap := conf.ToStringMap()
+	RemoveTLSRedacted(strMap)
+	return strMap, nil
+}
+
+func RemoveTLSRedacted(stringMap map[string]interface{}) {
+	type Node struct {
+		isTLSParent bool
+		data        map[string]interface{}
+	}
+	root := Node{isTLSParent: false, data: stringMap}
+	queue := []Node{root}
+	// Using BFS search through string Map and find sub settings of TLS
+	// Then delete REDACTED settings under TLS
+	for len(queue) > 0 {
+		node := queue[0]
+		queue = queue[1:]
+		for key, child := range node.data {
+			if childMap, ok := child.(map[string]interface{}); ok {
+				queue = append(queue, Node{key == common.TlsKey, childMap})
+			} else if child == "[REDACTED]" && node.isTLSParent {
+				delete(node.data, key)
+			}
+		}
+	}
 }
 
 func ConfigToTomlFile(config interface{}, tomlConfigFilePath string) error {

translator/config/schema.json

@@ -323,6 +323,29 @@
           },
           "additionalProperties": false
         },
+        "tlsDefinitions": {
+          "type": "object",
+          "properties": {
+            "ca_file": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 255
+            },
+            "cert_file": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 255
+            },
+            "key_file": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 255
+            },
+            "insecure": {
+              "type": "boolean"
+            }
+          }
+        },
         "swapDefinitions": {
           "$ref": "#/definitions/metricsDefinition/definitions/basicMetricDefinition"
         },
@@ -590,6 +613,9 @@
                   }
                 }
               },
+              "tls": {
+                "$ref": "#/definitions/metricsDefinition/definitions/tlsDefinitions"
+              },
               "additionalProperties": true
             },
             "ecs": {
@@ -913,7 +939,10 @@
               "$ref": "#/definitions/tracesDefinition/definitions/xrayDefinition"
             },
             "otlp": {
-              "$ref": "#/definitions/tracesDefinition/definitions/otlpDefinition"
+              "tls": {
+                "$ref": "#/definitions/metricsDefinition/definitions/tlsDefinitions"
+              },
+              "$ref": "#/definitions/tracesDefinition/definitions/otlpDefinitions"
             }
           },
           "minProperties": 1,
@@ -977,19 +1006,20 @@
           },
           "additionalProperties": false
         },
-        "otlpDefinition": {
-          "type": "object",
-          "properties": {
-            "grpc_endpoint": {
-              "description": "gRPC endpoint to use to listen for OTLP protobuf traces",
-              "$ref": "#/definitions/endpointOverrideDefinition"
+        "otlpDefinitions": {
+          "oneOf": [
+            {
+              "type": "array",
+              "minItems": 1,
+              "maxItems": 255,
+              "items": {
+                "$ref": "#/definitions/otlpObjectDefinition"
+              }
             },
-            "http_endpoint": {
-              "description": "HTTP endpoint to use to listen for OTLP JSON traces",
-              "$ref": "#/definitions/endpointOverrideDefinition"
+            {
+              "$ref": "#/definitions/otlpObjectDefinition"
             }
-          },
-          "additionalProperties": false
+          ]
         }
       }
     },
@@ -1045,6 +1075,20 @@
       },
       "additionalProperties": false
     },
+    "otlpObjectDefinition": {
+      "type": "object",
+      "properties": {
+        "grpc_endpoint": {
+          "description": "gRPC endpoint to use to listen for OTLP protobuf information",
+          "$ref": "#/definitions/endpointOverrideDefinition"
+        },
+        "http_endpoint": {
+          "description": "HTTP endpoint to use to listen for OTLP JSON information",
+          "$ref": "#/definitions/endpointOverrideDefinition"
+        }
+      },
+      "additionalProperties": false
+    },
     "ecsServiceDiscoveryDefinition": {
       "type": "object",
       "descriptions": "Define ECS service discovery for Prometheus",

translator/tocwconfig/sampleConfig/appsignals_and_eks_config.json

@@ -5,6 +5,10 @@
   "logs": {
     "metrics_collected": {
       "app_signals": {
+        "tls": {
+          "cert_file": "path/to/cert.crt",
+          "key_file": "path/to/key.key"
+        },
         "hosted_in": "TestCluster",
         "limiter": {
           "log_dropped_metrics": true,

translator/tocwconfig/sampleConfig/appsignals_and_eks_config.yaml

@@ -260,11 +260,11 @@ exporters:
             - aws.local.operation
             - aws.remote.service
             - aws.remote.operation
-            - HostedIn.EKS.Cluster
             - HostedIn.K8s.Namespace
             - K8s.RemoteNamespace
             - aws.remote.target
             - HostedIn.Environment
+            - HostedIn.EKS.Cluster
         local_mode: false
         max_retries: 2
         middleware: agenthealth/traces
@@ -559,13 +559,10 @@ processors:
                     enabled: true
             tls:
                 ca_file: ""
-                ca_pem: '[REDACTED]'
                 cert_file: ""
-                cert_pem: '[REDACTED]'
                 insecure: false
                 insecure_skip_verify: false
                 key_file: ""
-                key_pem: '[REDACTED]'
                 max_version: ""
                 min_version: ""
                 reload_interval: 0s
@@ -636,7 +633,15 @@ receivers:
                 max_concurrent_streams: 0
                 max_recv_msg_size_mib: 0
                 read_buffer_size: 524288
-                tls: null
+                tls:
+                  cert_file: path/to/cert.crt
+                  key_file: path/to/key.key
+                  ca_file: ""
+                  client_ca_file: ""
+                  client_ca_file_reload: false
+                  max_version: ""
+                  min_version: ""
+                  reload_interval: 0s
                 transport: tcp
                 write_buffer_size: 0
             http:
@@ -648,7 +653,15 @@ receivers:
                 max_request_body_size: 0
                 metrics_url_path: /v1/metrics
                 response_headers: {}
-                tls: null
+                tls:
+                  cert_file: path/to/cert.crt
+                  key_file: path/to/key.key
+                  client_ca_file: ""
+                  client_ca_file_reload: false
+                  ca_file: ""
+                  max_version: ""
+                  min_version: ""
+                  reload_interval: 0s
                 traces_url_path: /v1/traces
 service:
     extensions:

translator/tocwconfig/sampleConfig/complete_darwin_config.yaml

@@ -149,19 +149,16 @@ receivers:
             role_arn: trace_role_arn_value_test
             tls:
                 ca_file: ""
-                ca_pem: '[REDACTED]'
                 cert_file: ""
-                cert_pem: '[REDACTED]'
                 insecure: true
                 insecure_skip_verify: false
                 key_file: ""
-                key_pem: '[REDACTED]'
                 max_version: ""
                 min_version: ""
                 reload_interval: 0s
                 server_name_override: ""
         transport: udp
-    otlp:
+    otlp/traces:
         protocols:
             grpc:
                 auth: null
@@ -295,7 +292,7 @@ service:
                 - batch/xray
             receivers:
                 - awsxray
-                - otlp
+                - otlp/traces
     telemetry:
         logs:
             development: false

translator/tocwconfig/sampleConfig/complete_linux_config.yaml

@@ -152,19 +152,16 @@ receivers:
             role_arn: trace_role_arn_value_test
             tls:
                 ca_file: ""
-                ca_pem: '[REDACTED]'
                 cert_file: ""
-                cert_pem: '[REDACTED]'
                 insecure: true
                 insecure_skip_verify: false
                 key_file: ""
-                key_pem: '[REDACTED]'
                 max_version: ""
                 min_version: ""
                 reload_interval: 0s
                 server_name_override: ""
         transport: udp
-    otlp:
+    otlp/traces:
         protocols:
             grpc:
                 auth: null
@@ -298,7 +295,7 @@ service:
                 - batch/xray
             receivers:
                 - awsxray
-                - otlp
+                - otlp/traces
     telemetry:
         logs:
             development: false

translator/tocwconfig/sampleConfig/complete_windows_config.yaml

@@ -135,19 +135,16 @@ receivers:
             role_arn: trace_role_arn_value_test
             tls:
                 ca_file: ""
-                ca_pem: "[REDACTED]"
                 cert_file: ""
-                cert_pem: "[REDACTED]"
                 insecure: true
                 insecure_skip_verify: false
                 key_file: ""
-                key_pem: "[REDACTED]"
                 max_version: ""
                 min_version: ""
                 reload_interval: 0s
                 server_name_override: ""
         transport: udp
-    otlp:
+    otlp/traces:
         protocols:
             grpc:
                 auth: null
@@ -269,7 +266,7 @@ service:
                 - batch/xray
             receivers:
                 - awsxray
-                - otlp
+                - otlp/traces
     telemetry:
         logs:
             development: false

translator/tocwconfig/sampleConfig/trace_config_linux.yaml

@@ -47,7 +47,7 @@ receivers:
             region: us-west-2
             role_arn: ""
         transport: udp
-    otlp:
+    otlp/traces:
         protocols:
             grpc:
                 auth: null
@@ -82,7 +82,7 @@ service:
                 - batch/xray
             receivers:
                 - awsxray
-                - otlp
+                - otlp/traces
     telemetry:
         logs:
             development: false

translator/tocwconfig/sampleConfig/trace_config_windows.yaml

@@ -47,7 +47,7 @@ receivers:
             region: us-west-2
             role_arn: ""
         transport: udp
-    otlp:
+    otlp/traces:
         protocols:
             grpc:
                 auth: null
@@ -82,7 +82,7 @@ service:
                 - batch/xray
             receivers:
                 - awsxray
-                - otlp
+                - otlp/traces
     telemetry:
         logs:
             development: false

translator/translate/otel/common/common.go

@@ -53,6 +53,7 @@ const (
 	ServiceAddress                     = "service_address"
 	Udp                                = "udp"
 	Tcp                                = "tcp"
+	TlsKey                             = "tls"
 	Region                             = "region"
 	LogGroupName                       = "log_group_name"
 	LogStreamName                      = "log_stream_name"

translator/translate/otel/pipeline/host/translator.go

@@ -15,6 +15,7 @@ import (
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/ec2taggerprocessor"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metricsdecorator"
+	otlpReceiver "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp"
 )
 
 type translator struct {
@@ -42,7 +43,23 @@ func (t translator) ID() component.ID {
 func (t translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) {
 	if conf == nil || !conf.IsSet(common.MetricsKey) {
 		return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.MetricsKey}
-	} else if t.receivers.Len() == 0 {
+	}
+
+	hostReceivers := t.receivers
+	if common.PipelineNameHost == t.name {
+		switch v := conf.Get(common.ConfigKey(common.MetricsKey, common.MetricsCollectedKey, common.OtlpKey)).(type) {
+		case []interface{}:
+			for index, _ := range v {
+				hostReceivers.Set(otlpReceiver.NewTranslator(
+					otlpReceiver.WithDataType(component.DataTypeMetrics),
+					otlpReceiver.WithInstanceNum(index)))
+			}
+		case map[string]interface{}:
+			hostReceivers.Set(otlpReceiver.NewTranslator(otlpReceiver.WithDataType(component.DataTypeMetrics)))
+		}
+	}
+
+	if hostReceivers.Len() == 0 {
 		log.Printf("D! pipeline %s has no receivers", t.name)
 		return nil, nil
 	}