From c0de3c1b93b18d492d76b8fa87a58846c0b796d5 Mon Sep 17 00:00:00 2001 From: Andrew Huynh <17056456+SaxyPandaBear@users.noreply.github.com> Date: Tue, 9 May 2023 10:23:41 -0400 Subject: [PATCH] Start EKS test infra (#209) * added EKS terraform set up * add generator config * fix security groups config * add EKS cluster policy * try to fix eks cluster startup issue * fix provider * try fixing connection refused error * iterating * add more k8s configs * add cluster auth config * add namespace and depends chaining * fix namespace config * add label selector * added basic eks test runner * fix terraform * add node group * add aws auth config * syntax * Fix syntax * try to get node groups to attach in example * fix syntax * try running cwa * fix resource creation order * add logging? * add cwagent test policy to node * use managed policy * update docs, implement validation * update cluster and node naming * fix test to query ClusterName properly * add license header * make fmt for terraform * add todos, update doc for EKS permissions, formatting * reintroduce node role in terraform * update from minor code review comments --------- Co-authored-by: Andrew Huynh --- docs/getting-start.md | 401 +++++++++-------- environment/metadata.go | 37 +- .../resources/eks_daemon_test_matrix.json | 5 + generator/test_case_generator.go | 3 + go.mod | 6 +- go.sum | 9 +- terraform/eks/daemon/main.tf | 426 ++++++++++++++++++ terraform/eks/daemon/providers.tf | 17 + terraform/eks/daemon/variables.tf | 22 + terraform/setup/iam.tf | 15 + terraform/setup/vpc.tf | 1 - test/metric/dimension/instanceid_provider.go | 25 + test/metric/dimension/provider.go | 1 + .../eks_daemonset_test.go | 102 +++++ .../metric_value_benchmark/eks_test_runner.go | 32 ++ .../metrics_value_benchmark_test.go | 29 +- 16 files changed, 913 insertions(+), 218 deletions(-) create mode 100644 generator/resources/eks_daemon_test_matrix.json create mode 100644 terraform/eks/daemon/main.tf create mode 100644 terraform/eks/daemon/providers.tf create mode 100644 terraform/eks/daemon/variables.tf create mode 100644 test/metric_value_benchmark/eks_daemonset_test.go create mode 100644 test/metric_value_benchmark/eks_test_runner.go diff --git a/docs/getting-start.md b/docs/getting-start.md index 4a77d26b4..6bb01a547 100644 --- a/docs/getting-start.md +++ b/docs/getting-start.md @@ -79,210 +79,221 @@ required for testing locally but is required for testing on your personal fork. 1. Navigate to CloudFormation from aws console 2. Choose file upload. 3. Upload the following as a file -``` -Parameters: - GitHubOrg: - Type: String - RepositoryName: - Type: String - OIDCProviderArn: - Description: Arn for the GitHub OIDC Provider. - Default: "" - Type: String - -Conditions: - CreateOIDCProvider: !Equals - - !Ref OIDCProviderArn - - "" - -Resources: - Role: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Action: sts:AssumeRoleWithWebIdentity - Principal: - Federated: !If - - CreateOIDCProvider - - !Ref GithubOidc - - !Ref OIDCProviderArn - Condition: - StringLike: - token.actions.githubusercontent.com:sub: !Sub repo:${GitHubOrg}/${RepositoryName}:* - - GithubOidc: - Type: AWS::IAM::OIDCProvider - Condition: CreateOIDCProvider - Properties: - Url: https://token.actions.githubusercontent.com - ClientIdList: - - sts.amazonaws.com - ThumbprintList: - - 6938fd4d98bab03faadb97b34396831e3780aea1 - -Outputs: - Role: - Value: !GetAtt Role.Arn -``` + ``` + Parameters: + GitHubOrg: + Type: String + RepositoryName: + Type: String + OIDCProviderArn: + Description: Arn for the GitHub OIDC Provider. + Default: "" + Type: String + AccountRoot: + Description: "This account's ID to allow assume, in the form arn:aws:iam:[account ID]:root" + Type: String + + Conditions: + CreateOIDCProvider: !Equals + - !Ref OIDCProviderArn + - "" + + Resources: + Role: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Action: sts:AssumeRoleWithWebIdentity + Principal: + Federated: !If + - CreateOIDCProvider + - !Ref GithubOidc + - !Ref OIDCProviderArn + Condition: + StringLike: + token.actions.githubusercontent.com:sub: !Sub repo:${GitHubOrg}/${RepositoryName}:* + - Sid: AssumeFromRoot + Effect: Allow + Action: sts:AssumeRole + Principal: + AWS: !Ref AccountRoot + - Sid: EKS + Effect: Allow + Action: sts:AssumeRole + Principal: + Service: + - eks.amazonaws.com + + GithubOidc: + Type: AWS::IAM::OIDCProvider + Condition: CreateOIDCProvider + Properties: + Url: https://token.actions.githubusercontent.com + ClientIdList: + - sts.amazonaws.com + ThumbprintList: + - 6938fd4d98bab03faadb97b34396831e3780aea1 + + Outputs: + Role: + Value: !GetAtt Role.Arn + ``` 4. the UI should ask you for inputs for the parameters. In `GitHubOrg`, type in your github username. In `RepositoryName`, type in your fork repo's name. e.g. amazon-cloudwatch-agent -5. Choose a stackname. Anything. e.g. Terraform-IntegTest-Role -6. After creating the stack, navigate to the `Resources` tab of the created stack -7. Click on the role ID that was created by CloudFormation -8. Click add permission -9. Click attach policy, and then click create policy. -10. Click JSON tab and copy and paste the following - -```json -{ - "Version": "2012-10-17", - "Statement": [ +5. For the `AccountRoot`, input your root ARN, e.g.: `arn:aws:iam::1234567890:root` +6. Choose a stack name. Anything. e.g. Terraform-IntegTest-Role +7. After creating the stack, navigate to the `Resources` tab of the created stack +8. Click on the role ID that was created by CloudFormation +9. Click add permission +10. Click attach policy, and then click create policy. +11. Click JSON tab and copy and paste the following + ```json { - "Effect": "Allow", - "Action": [ - "autoscaling:CreateAutoScalingGroup", - "autoscaling:CreateLaunchConfiguration", - "autoscaling:CreateOrUpdateTags", - "autoscaling:DeleteAutoScalingGroup", - "autoscaling:DeleteLaunchConfiguration", - "autoscaling:DeleteTags", - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeScalingActivities", - "autoscaling:DescribeTags", - "autoscaling:SetInstanceProtection", - "autoscaling:UpdateAutoScalingGroup", - "cloudwatch:GetMetricData", - "dynamodb:*", - "ec2:AuthorizeSecurityGroupEgress", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateNetworkInterface", - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:CreateTags", - "ec2:DeleteKeyPair", - "ec2:DeleteNetworkInterface", - "ec2:DeleteSecurityGroup", - "ec2:DescribeAccountAttributes", - "ec2:DescribeImages", - "ec2:DescribeInstanceAttribute", - "ec2:DescribeInstanceCreditSpecifications", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstances", - "ec2:DescribeKeyPairs", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeRouteTables", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeTags", - "ec2:DescribeVolumes", - "ec2:DescribeVpcAttribute", - "ec2:DescribeVpcs", - "ec2:GetPasswordData", - "ec2:ImportKeyPair", - "ec2:ModifyInstanceAttribute", - "ec2:RevokeSecurityGroupEgress", - "ec2:RunInstances", - "ec2:RunInstances", - "ec2:TerminateInstances", - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:CompleteLayerUpload", - "ecr:DescribeImages", - "ecr:DescribeRepositories", - "ecr:GetAuthorizationToken", - "ecr:InitiateLayerUpload", - "ecr:ListImages", - "ecr:ListTagsForResource", - "ecr:PutImage", - "ecr:UploadLayerPart", - "ecs:CreateCapacityProvider", - "ecs:CreateCluster", - "ecs:CreateService", - "ecs:CreateTaskSet", - "ecs:DeleteCapacityProvider", - "ecs:DeleteCluster", - "ecs:DeleteService", - "ecs:DeleteTaskSet", - "ecs:DeregisterTaskDefinition", - "ecs:DescribeCapacityProviders", - "ecs:DescribeClusters", - "ecs:DescribeContainerInstances", - "ecs:DescribeServices", - "ecs:DescribeTaskDefinition", - "ecs:DescribeTasks", - "ecs:ListClusters", - "ecs:ListContainerInstances", - "ecs:ListServices", - "ecs:ListTagsForResource", - "ecs:ListTaskDefinitions", - "ecs:ListTasks", - "ecs:PutClusterCapacityProviders", - "ecs:RegisterTaskDefinition", - "ecs:RunTask", - "ecs:StartTask", - "ecs:StopTask", - "ecs:UpdateService", - "iam:AddRoleToInstanceProfile", - "iam:AttachRolePolicy", - "iam:CreateInstanceProfile", - "iam:CreatePolicy", - "iam:CreateRole", - "iam:CreateServiceLinkedRole", - "iam:DeleteInstanceProfile", - "iam:DeletePolicy", - "iam:DeleteRole", - "iam:DetachRolePolicy", - "iam:GetInstanceProfile", - "iam:GetPolicy", - "iam:GetPolicyVersion", - "iam:GetRole", - "iam:ListAttachedRolePolicies", - "iam:ListInstanceProfilesForRole", - "iam:ListPolicyVersions", - "iam:ListRolePolicies", - "iam:PassRole", - "iam:RemoveRoleFromInstanceProfile", - "logs:CreateLogGroup", - "logs:DeleteLogGroup", - "logs:DescribeLogGroups", - "logs:ListTagsForResource", - "logs:ListTagsLogGroup", - "s3:DeleteObject", - "s3:GetObject", - "s3:GetObjectAcl", - "s3:ListBucket", - "s3:PutObject", - "ssm:DeleteParameter", - "ssm:DeleteParameters", - "ssm:DescribeParameters", - "ssm:GetParameter", - "ssm:GetParameters", - "ssm:ListTagsForResource", - "ssm:PutParameter", - "sts:GetCallerIdentity" - ], - "Resource": "*" + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:CreateAutoScalingGroup", + "autoscaling:CreateLaunchConfiguration", + "autoscaling:CreateOrUpdateTags", + "autoscaling:DeleteAutoScalingGroup", + "autoscaling:DeleteLaunchConfiguration", + "autoscaling:DeleteTags", + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", + "autoscaling:DescribeTags", + "autoscaling:SetInstanceProtection", + "autoscaling:UpdateAutoScalingGroup", + "cloudwatch:GetMetricData", + "dynamodb:*", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateNetworkInterface", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:DeleteNetworkInterface", + "ec2:DeleteSecurityGroup", + "ec2:DescribeAccountAttributes", + "ec2:DescribeImages", + "ec2:DescribeInstanceAttribute", + "ec2:DescribeInstanceCreditSpecifications", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeKeyPairs", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeVpcAttribute", + "ec2:DescribeVpcs", + "ec2:GetPasswordData", + "ec2:ImportKeyPair", + "ec2:ModifyInstanceAttribute", + "ec2:RevokeSecurityGroupEgress", + "ec2:RunInstances", + "ec2:RunInstances", + "ec2:TerminateInstances", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:CompleteLayerUpload", + "ecr:DescribeImages", + "ecr:DescribeRepositories", + "ecr:GetAuthorizationToken", + "ecr:InitiateLayerUpload", + "ecr:ListImages", + "ecr:ListTagsForResource", + "ecr:PutImage", + "ecr:UploadLayerPart", + "ecs:CreateCapacityProvider", + "ecs:CreateCluster", + "ecs:CreateService", + "ecs:CreateTaskSet", + "ecs:DeleteCapacityProvider", + "ecs:DeleteCluster", + "ecs:DeleteService", + "ecs:DeleteTaskSet", + "ecs:DeregisterTaskDefinition", + "ecs:DescribeCapacityProviders", + "ecs:DescribeClusters", + "ecs:DescribeContainerInstances", + "ecs:DescribeServices", + "ecs:DescribeTaskDefinition", + "ecs:DescribeTasks", + "ecs:ListClusters", + "ecs:ListContainerInstances", + "ecs:ListServices", + "ecs:ListTagsForResource", + "ecs:ListTaskDefinitions", + "ecs:ListTasks", + "ecs:PutClusterCapacityProviders", + "ecs:RegisterTaskDefinition", + "ecs:RunTask", + "ecs:StartTask", + "ecs:StopTask", + "ecs:UpdateService", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:CreateServiceLinkedRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "logs:CreateLogGroup", + "logs:DeleteLogGroup", + "logs:DescribeLogGroups", + "logs:ListTagsForResource", + "logs:ListTagsLogGroup", + "s3:DeleteObject", + "s3:GetObject", + "s3:GetObjectAcl", + "s3:ListBucket", + "s3:PutObject", + "ssm:DeleteParameter", + "ssm:DeleteParameters", + "ssm:DescribeParameters", + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:ListTagsForResource", + "ssm:PutParameter", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] } - ] -} -``` - -11. Once creation is done, go back to the IAM role and attach the policy you just created by searching for the policy name. -12. Follow [docs](https://docs.github.com/en/actions/security-guides/encrypted-secrets) to add TERRAFORM_AWS_ASSUME_ROLE as GitHub repository secret. Name is `TERRAFORM_AWS_ASSUME_ROLE` and Secret is the IAM role's ARN. - - - + ``` +12. Once creation is done, go back to the IAM role and attach the policy you just created by searching for the policy name. +13. For EKS testing, attach the `AmazonEKSClusterPolicy` to the terraform role as well. +14. Follow [docs](https://docs.github.com/en/actions/security-guides/encrypted-secrets) to add TERRAFORM_AWS_ASSUME_ROLE as GitHub repository secret. Name is `TERRAFORM_AWS_ASSUME_ROLE` and Secret is the IAM role's ARN. ### Notes [reference of how to create role](https://github.com/aws-actions/configure-aws-credentials) -## 7. Setup the integration test resources +## 7. Set up the integration test resources 1. Navigate to your fork -2. Run the following command to setup all the resources being used for integration test (e.g vpc, iam, etc) +2. Run the following command to set up all the resources being used for integration test (e.g vpc, iam, etc) ```shell cd terraform/setup # assuming you are still in the ./terraform terraform init diff --git a/environment/metadata.go b/environment/metadata.go index 5489689e9..38882c560 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -27,6 +27,7 @@ type MetaData struct { S3Key string CwaCommitSha string CaCertPath string + EKSClusterName string } type MetaDataStrings struct { @@ -41,6 +42,7 @@ type MetaDataStrings struct { S3Key string CwaCommitSha string CaCertPath string + EKSClusterName string } func registerComputeType(dataString *MetaDataStrings) { @@ -67,22 +69,25 @@ func registerECSData(dataString *MetaDataStrings) { flag.StringVar(&(dataString.EcsServiceName), "cwagentECSServiceName", "", "Used to restart ecs task to apply new agent config") } +func registerEKSData(d *MetaDataStrings) { + flag.StringVar(&(d.EKSClusterName), "eksClusterName", "", "EKS cluster name") +} + func registerPluginTestsToExecute(dataString *MetaDataStrings) { flag.StringVar(&(dataString.EC2PluginTests), "plugins", "", "Comma-delimited list of plugins to test. Default is empty, which tests all") } -func fillComputeType(e *MetaData, data *MetaDataStrings) *MetaData { +func fillComputeType(e *MetaData, data *MetaDataStrings) { computeType, ok := computetype.FromString(data.ComputeType) if !ok { log.Panic("Invalid compute type. Needs to be EC2/ECS/EKS. Compute Type is a required flag. :" + data.ComputeType) } e.ComputeType = computeType - return e } -func fillECSData(e *MetaData, data *MetaDataStrings) *MetaData { +func fillECSData(e *MetaData, data *MetaDataStrings) { if e.ComputeType != computetype.ECS { - return e + return } ecsLaunchType, ok := ecslaunchtype.FromString(data.EcsLaunchType) @@ -103,18 +108,16 @@ func fillECSData(e *MetaData, data *MetaDataStrings) *MetaData { e.CwagentConfigSsmParamName = data.CwagentConfigSsmParamName e.EcsServiceName = data.EcsServiceName e.EcsClusterName = awsservice.GetClusterName(data.EcsClusterArn) - - return e } -func fillEC2PluginTests(e *MetaData, data *MetaDataStrings) *MetaData { +func fillEC2PluginTests(e *MetaData, data *MetaDataStrings) { if e.ComputeType != computetype.EC2 { - return e + return } if len(data.EC2PluginTests) == 0 { log.Println("Testing all EC2 plugins") - return e + return } plugins := strings.Split(strings.ReplaceAll(data.EC2PluginTests, " ", ""), ",") @@ -124,13 +127,20 @@ func fillEC2PluginTests(e *MetaData, data *MetaDataStrings) *MetaData { m[strings.ToLower(p)] = struct{}{} } e.EC2PluginTests = m +} + +func fillEKSData(e *MetaData, data *MetaDataStrings) { + if e.ComputeType != computetype.EKS { + return + } - return e + e.EKSClusterName = data.EKSClusterName } func RegisterEnvironmentMetaDataFlags(metaDataStrings *MetaDataStrings) *MetaDataStrings { registerComputeType(metaDataStrings) registerECSData(metaDataStrings) + registerEKSData(metaDataStrings) registerBucket(metaDataStrings) registerS3Key(metaDataStrings) registerCwaCommitSha(metaDataStrings) @@ -141,9 +151,10 @@ func RegisterEnvironmentMetaDataFlags(metaDataStrings *MetaDataStrings) *MetaDat func GetEnvironmentMetaData(data *MetaDataStrings) *MetaData { metaData := &(MetaData{}) - metaData = fillComputeType(metaData, data) - metaData = fillECSData(metaData, data) - metaData = fillEC2PluginTests(metaData, data) + fillComputeType(metaData, data) + fillECSData(metaData, data) + fillEKSData(metaData, data) + fillEC2PluginTests(metaData, data) metaData.Bucket = data.Bucket metaData.S3Key = data.S3Key metaData.CwaCommitSha = data.CwaCommitSha diff --git a/generator/resources/eks_daemon_test_matrix.json b/generator/resources/eks_daemon_test_matrix.json new file mode 100644 index 000000000..81ed61090 --- /dev/null +++ b/generator/resources/eks_daemon_test_matrix.json @@ -0,0 +1,5 @@ +[ + { + + } +] \ No newline at end of file diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index e302a0d69..80a6fbcc0 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -76,6 +76,9 @@ var testTypeToTestDirMap = map[string][]string{ "ec2_acceptance": { "./test/acceptance", }, + "eks_daemon": { + "./test/metric_value_benchmark", + }, } func main() { diff --git a/go.mod b/go.mod index b3e98b13d..0cbe369c0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( collectd.org v0.5.0 github.com/DataDog/datadog-go v4.8.3+incompatible github.com/aws/aws-sdk-go v1.44.188 - github.com/aws/aws-sdk-go-v2 v1.17.4 + github.com/aws/aws-sdk-go-v2 v1.18.0 github.com/aws/aws-sdk-go-v2/config v1.18.10 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 @@ -38,8 +38,8 @@ require ( github.com/Microsoft/go-winio v0.6.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 // indirect diff --git a/go.sum b/go.sum index 46c03740e..8c5e94d2e 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,9 @@ github.com/aws/aws-sdk-go v1.44.188/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8 github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.17.4 h1:wyC6p9Yfq6V2y98wfDsj6OnNQa4w2BLGCLIxzNhwOGY= github.com/aws/aws-sdk-go-v2 v1.17.4/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= +github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= github.com/aws/aws-sdk-go-v2/config v1.18.10 h1:Znce11DWswdh+5kOsIp+QaNfY9igp1QUN+fZHCKmeCI= @@ -24,13 +25,15 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.49/go.mod h1:N9gSChQkKpdAj github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 h1:r+XwaCLpIvCKjBIYy/HVZujQS9tsz5ohHG3ZIe0wKoE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 h1:7AwGYXDdqRQYsluvKFmWoqpcOQJ4bH634SkYf3FNj/A= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.18 h1:H/mF2LNWwX00lD6FlYfKpLLZgUW7oIzCBkig78x4Xok= diff --git a/terraform/eks/daemon/main.tf b/terraform/eks/daemon/main.tf new file mode 100644 index 000000000..bc0c36ae6 --- /dev/null +++ b/terraform/eks/daemon/main.tf @@ -0,0 +1,426 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +module "common" { + source = "../../common" + cwagent_image_repo = var.cwagent_image_repo + cwagent_image_tag = var.cwagent_image_tag +} + +module "basic_components" { + source = "../../basic_components" + + region = var.region +} + +data "aws_eks_cluster_auth" "this" { + name = aws_eks_cluster.this.name +} + +resource "aws_eks_cluster" "this" { + name = "cwagent-eks-integ-${module.common.testing_id}" + role_arn = module.basic_components.role_arn + version = "1.24" # TODO: parameterize the EKS version + enabled_cluster_log_types = [ + "api", + "audit", + "authenticator", + "controllerManager", + "scheduler" + ] + vpc_config { + subnet_ids = module.basic_components.public_subnet_ids + security_group_ids = [module.basic_components.security_group] + } +} + +# EKS Node Groups +resource "aws_eks_node_group" "this" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "cwagent-eks-integ-node" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = module.basic_components.public_subnet_ids + + scaling_config { + desired_size = 1 + max_size = 1 + min_size = 1 + } + + ami_type = "AL2_x86_64" + capacity_type = "ON_DEMAND" + disk_size = 20 + instance_types = ["t3.medium"] + + depends_on = [ + aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy, + aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy + ] +} + +# EKS Node IAM Role +resource "aws_iam_role" "node_role" { + name = "cwagent-eks-Worker-Role-${module.common.testing_id}" + + assume_role_policy = < 0 { + log.Println("failed to get dimensions") + return testResult + } + + fetcher := metric.MetricValueFetcher{} + values, err := fetcher.Fetch("ContainerInsights", name, dims, metric.AVERAGE, test_runner.HighResolutionStatPeriod) + if err != nil { + log.Println("failed to fetch metrics", err) + return testResult + } + + if !isAllValuesGreaterThanOrEqualToExpectedValue(name, values, 0) { + return testResult + } + + testResult.Status = status.SUCCESSFUL + return testResult +} + +func (e *EKSDaemonTestRunner) GetTestName() string { + return "EKSContainerInstance" +} + +func (e *EKSDaemonTestRunner) GetAgentConfigFileName() string { + return "" // TODO: maybe not needed? +} + +func (e *EKSDaemonTestRunner) GetAgentRunDuration() time.Duration { + return time.Minute * 3 +} + +func (e *EKSDaemonTestRunner) GetMeasuredMetrics() []string { + return []string{ + "node_cpu_reserved_capacity", + "node_cpu_utilization", + "node_network_total_bytes", + "node_filesystem_utilization", + "node_number_of_running_pods", + "node_number_of_running_containers", + "node_memory_utilization", + "node_memory_reserved_capacity", + } +} + +func (e *EKSDaemonTestRunner) SetupBeforeAgentRun() error { + return nil +} + +func (e *EKSDaemonTestRunner) SetupAfterAgentRun() error { + return nil +} + +var _ test_runner.ITestRunner = (*EKSDaemonTestRunner)(nil) diff --git a/test/metric_value_benchmark/eks_test_runner.go b/test/metric_value_benchmark/eks_test_runner.go new file mode 100644 index 000000000..86598ff5e --- /dev/null +++ b/test/metric_value_benchmark/eks_test_runner.go @@ -0,0 +1,32 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package metric_value_benchmark + +import ( + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" + "log" + "time" +) + +type EKSTestRunner struct { + runner test_runner.ITestRunner + env environment.MetaData +} + +func (t *EKSTestRunner) Run(s test_runner.ITestSuite, e *environment.MetaData) { + name := t.runner.GetTestName() + log.Printf("Running %s", name) + dur := t.runner.GetAgentRunDuration() + time.Sleep(dur) + + res := t.runner.Validate() + s.AddToSuiteResult(res) + if res.GetStatus() != status.SUCCESSFUL { + log.Printf("%s test group failed", name) + } +} diff --git a/test/metric_value_benchmark/metrics_value_benchmark_test.go b/test/metric_value_benchmark/metrics_value_benchmark_test.go index 1d50e0f3c..d4fa65735 100644 --- a/test/metric_value_benchmark/metrics_value_benchmark_test.go +++ b/test/metric_value_benchmark/metrics_value_benchmark_test.go @@ -45,6 +45,7 @@ func init() { var ( ecsTestRunners []*ECSTestRunner ec2TestRunners []*test_runner.TestRunner + eksTestRunners []*EKSTestRunner ) func getEcsTestRunners(env *environment.MetaData) []*ECSTestRunner { @@ -62,6 +63,21 @@ func getEcsTestRunners(env *environment.MetaData) []*ECSTestRunner { return ecsTestRunners } +func getEksTestRunners(env *environment.MetaData) []*EKSTestRunner { + if eksTestRunners == nil { + factory := dimension.GetDimensionFactory(*env) + eksTestRunners = []*EKSTestRunner{ + { + runner: &EKSDaemonTestRunner{BaseTestRunner: test_runner.BaseTestRunner{ + DimensionFactory: factory, + }}, + env: *env, + }, + } + } + return eksTestRunners +} + func getEc2TestRunners(env *environment.MetaData) []*test_runner.TestRunner { if ec2TestRunners == nil { factory := dimension.GetDimensionFactory(*env) @@ -87,12 +103,19 @@ func getEc2TestRunners(env *environment.MetaData) []*test_runner.TestRunner { func (suite *MetricBenchmarkTestSuite) TestAllInSuite() { env := environment.GetEnvironmentMetaData(envMetaDataStrings) - if env.ComputeType == computetype.ECS { - log.Print("Environment compute type is ECS") + switch env.ComputeType { + case computetype.ECS: + log.Println("Environment compute type is ECS") for _, ecsTestRunner := range getEcsTestRunners(env) { ecsTestRunner.Run(suite, env) } - } else { + case computetype.EKS: + log.Println("Environment compute type is EKS") + for _, testRunner := range getEksTestRunners(env) { + testRunner.Run(suite, env) + } + default: // EC2 tests + log.Println("Environment compute type is EC2") for _, testRunner := range getEc2TestRunners(env) { if shouldRunEC2Test(env, testRunner) { testRunner.Run(suite)