Add EKS integration test for FluentBit log emission with Instrumentat…

…ion source (aws#429) * Add EKS integration test for FluentBit log emission with Instrumentation service name source * Modify test timeline to ensure entity query is ready * Fix linter errors
chadpatel · Nov 15, 2024 · 3a6e4dc · 3a6e4dc
1 parent 93dedc0
commit 3a6e4dc
Show file tree

Hide file tree

Showing 2 changed files with 138 additions and 8 deletions.
diff --git a/terraform/eks/daemon/entity/main.tf b/terraform/eks/daemon/entity/main.tf
@@ -11,6 +11,10 @@ module "basic_components" {
   region = var.region
 }
 
+locals {
+  aws_eks = "aws eks --region ${var.region}"
+}
+
 data "aws_eks_cluster_auth" "this" {
   name = aws_eks_cluster.this.name
 }
@@ -190,6 +194,34 @@ resource "helm_release" "aws_observability" {
   null_resource.clone_helm_chart]
 }
 
+resource "null_resource" "kubectl" {
+  depends_on = [
+    aws_eks_cluster.this,
+    aws_eks_node_group.this,
+  ]
+  provisioner "local-exec" {
+    command = <<-EOT
+      ${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name}
+      ${local.aws_eks} list-clusters --output text
+      ${local.aws_eks} describe-cluster --name ${aws_eks_cluster.this.name} --output text
+    EOT
+  }
+}
+
+resource "null_resource" "update_image" {
+  depends_on = [helm_release.aws_observability, null_resource.kubectl]
+  triggers = {
+    timestamp = "${timestamp()}" # Forces re-run on every apply
+  }
+  provisioner "local-exec" {
+    command = <<-EOT
+      kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]'
+      kubectl set image deployment/amazon-cloudwatch-observability-controller-manager -n amazon-cloudwatch manager=public.ecr.aws/cloudwatch-agent/cloudwatch-agent-operator:latest
+      sleep 10
+    EOT
+  }
+}
+
 resource "kubernetes_pod" "log_generator" {
   depends_on = [aws_eks_node_group.this]
   metadata {
@@ -210,6 +242,76 @@ resource "kubernetes_pod" "log_generator" {
   }
 }
 
+resource "kubernetes_pod" "petclinic_instrumentation" {
+  depends_on = [aws_eks_node_group.this, helm_release.aws_observability, null_resource.update_image]
+  metadata {
+    name = "petclinic-instrumentation-default-env"
+    annotations = {
+      "instrumentation.opentelemetry.io/inject-java" = "true"
+    }
+    labels = {
+      app = "petclinic"
+    }
+  }
+
+  spec {
+    container {
+      name  = "petclinic"
+      image = "506463145083.dkr.ecr.us-west-2.amazonaws.com/cwagent-integ-test-petclinic:latest"
+
+      port {
+        container_port = 8080
+      }
+
+      env {
+        name  = "OTEL_SERVICE_NAME"
+        value = "petclinic-custom-service-name"
+      }
+
+    }
+  }
+}
+
+# Traffic generator pod with bash command
+resource "kubernetes_pod" "traffic_generator_instrumentation" {
+  depends_on = [kubernetes_pod.petclinic_instrumentation, kubernetes_service.petclinic_service]
+  metadata {
+    name = "traffic-generator-instrumentation-default-env"
+  }
+
+  spec {
+    container {
+      name  = "traffic-generator"
+      image = "alpine"
+
+      # Run the curl command as a loop to repeatedly send requests
+      command = ["/bin/sh", "-c"]
+      args = [
+        "apk add --no-cache curl && while true; do curl -s http://petclinic-service:8080/client-call; sleep 1; done"
+      ]
+    }
+  }
+}
+
+
+# Service for Petclinic Pods to load-balance traffic
+resource "kubernetes_service" "petclinic_service" {
+  metadata {
+    name = "petclinic-service"
+  }
+
+  spec {
+    selector = {
+      app = "petclinic"
+    }
+
+    port {
+      port        = 8080
+      target_port = 8080
+    }
+  }
+}
+
 
 # Get the single instance ID of the node in the node group
 data "aws_instances" "eks_node" {
@@ -234,7 +336,10 @@ resource "null_resource" "validator" {
   depends_on = [
     aws_eks_node_group.this,
     helm_release.aws_observability,
-    kubernetes_pod.log_generator
+    null_resource.update_image,
+    kubernetes_pod.log_generator,
+    kubernetes_pod.petclinic_instrumentation,
+    kubernetes_pod.traffic_generator_instrumentation
   ]
 
   triggers = {

diff --git a/test/entity/entity_test.go b/test/entity/entity_test.go
@@ -19,7 +19,7 @@ import (
 )
 
 const (
-	sleepForFlush = 180 * time.Second
+	sleepForFlush = 240 * time.Second
 
 	entityType        = "@entity.KeyAttributes.Type"
 	entityName        = "@entity.KeyAttributes.Name"
@@ -32,6 +32,13 @@ const (
 	entityK8sNamespace      = "@entity.Attributes.K8s.Namespace"
 	entityK8sWorkload       = "@entity.Attributes.K8s.Workload"
 	entityServiceNameSource = "@entity.Attributes.AWS.ServiceNameSource"
+
+	// Constants for possible vaues for entity attributes
+	eksServiceEntityType                   = "Service"
+	entityEKSPlatform                      = "AWS::EKS"
+	k8sDefaultNamespace                    = "default"
+	entityServiceNameSourceInstrumentation = "Instrumentation"
+	entityServiceNameSourceK8sWorkload     = "K8sWorkload"
 )
 
 type EntityValidator struct {
@@ -160,16 +167,34 @@ func TestPutLogEventEntityEKS(t *testing.T) {
 			agentConfigPath: filepath.Join("resources", "compass_default_log.json"),
 			podName:         "log-generator",
 			expectedEntity: expectedEntity{
-				entityType:        "Service",
+				entityType:        eksServiceEntityType,
 				name:              "log-generator",
-				environment:       "eks:" + env.EKSClusterName + "/" + "default",
-				platformType:      "AWS::EKS",
+				environment:       "eks:" + env.EKSClusterName + "/" + k8sDefaultNamespace,
+				platformType:      entityEKSPlatform,
 				k8sWorkload:       "log-generator",
 				k8sNode:           *instancePrivateDNS,
-				k8sNamespace:      "default",
+				k8sNamespace:      k8sDefaultNamespace,
+				eksCluster:        env.EKSClusterName,
+				instanceId:        env.InstanceId,
+				serviceNameSource: entityServiceNameSourceK8sWorkload,
+			},
+		},
+		"Entity/InstrumentationServiceNameSource": {
+			agentConfigPath: filepath.Join("resources", "compass_default_log.json"),
+			podName:         "petclinic-instrumentation-default-env",
+			expectedEntity: expectedEntity{
+				entityType: eksServiceEntityType,
+				// This service name comes from OTEL_SERVICE_NAME which is
+				// customized in the terraform code when creating the pod
+				name:              "petclinic-custom-service-name",
+				environment:       "eks:" + env.EKSClusterName + "/" + k8sDefaultNamespace,
+				platformType:      entityEKSPlatform,
+				k8sWorkload:       "petclinic-instrumentation-default-env",
+				k8sNode:           *instancePrivateDNS,
+				k8sNamespace:      k8sDefaultNamespace,
 				eksCluster:        env.EKSClusterName,
 				instanceId:        env.InstanceId,
-				serviceNameSource: "K8sWorkload",
+				serviceNameSource: entityServiceNameSourceInstrumentation,
 			},
 		},
 	}
@@ -201,7 +226,7 @@ func ValidateLogEntity(t *testing.T, logGroup, logStream string, end *time.Time,
 		t.Fatalf("application log group used for entity validation doesn't exist: %s", logGroup)
 	}
 
-	begin := end.Add(-sleepForFlush * 2)
+	begin := end.Add(-2 * time.Minute)
 	log.Printf("Start time is %s and end time is %s", begin.String(), end.String())
 
 	result, err := awsservice.GetLogQueryResults(logGroup, begin.Unix(), end.Unix(), queryString)