DO NOT MERGE Test aws integrations on Localstack API

.buildkite/pipeline.trigger.integration.tests.sh

@@ -18,23 +18,24 @@ STACK_COMMAND_TESTS=(
     test-stack-command-8x
 )
 
-for test in ${STACK_COMMAND_TESTS[@]}; do
-    echo "      - label: \":go: Running integration test: ${test}\""
-    echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
-    echo "        agents:"
-    echo "          provider: \"gcp\""
-    echo "        artifact_paths:"
-    echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*"
-    echo "          - build/elastic-stack-status/*/*"
-done
+# for test in ${STACK_COMMAND_TESTS[@]}; do
+#     echo "      - label: \":go: Running integration test: ${test}\""
+#     echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
+#     echo "        agents:"
+#     echo "          provider: \"gcp\""
+#     echo "        artifact_paths:"
+#     echo "          - build/elastic-stack-dump/stack/*/logs/*.log"
+#     echo "          - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*"
+#     echo "          - build/elastic-stack-status/*/*"
+# done
 
 CHECK_PACKAGES_TESTS=(
     test-check-packages-other
     test-check-packages-with-kind
     test-check-packages-with-custom-agent
     test-check-packages-benchmarks
 )
+
 for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "      - label: \":go: Running integration test: ${test}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"

Makefile

@@ -81,7 +81,7 @@ test-check-packages-benchmarks:
 	PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh
 
 test-check-packages-parallel:
-	PACKAGE_TEST_TYPE=parallel ./scripts/test-check-packages.sh
+	PACKAGE_TEST_TYPE=parallel PACKAGE_UNDER_TEST=aws ./scripts/test-check-packages.sh
 
 test-check-packages-with-custom-agent:
 	PACKAGE_TEST_TYPE=with-custom-agent ./scripts/test-check-packages.sh

internal/testrunner/runners/system/servicedeployer/_static/localstack_deployer.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+services:
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
+    image: localstack/localstack
+    hostname: localstack
+    ports:
+      - "4566:4566"            # LocalStack Gateway
+    environment:
+      - SERVICES=sqs,sns
+      - DEBUG=1
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - HOST_TMP_FOLDER=${TMPDIR}
+      - HOSTNAME_EXTERNAL=localstack
+      - S3_HOSTNAME=localstack
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"

internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh

@@ -20,6 +20,7 @@ trap cleanup EXIT INT TERM
 
 terraform init
 terraform plan
+
 terraform apply -auto-approve
 
 terraform output -json > /output/tfOutputValues.json

internal/testrunner/runners/system/servicedeployer/terraform.go

@@ -23,12 +23,16 @@ import (
 const (
 	terraformDeployerDir        = "terraform"
 	terraformDeployerYml        = "terraform-deployer.yml"
+	localstackDeployerYml       = "localstack-deployer.yml"
 	terraformDeployerDockerfile = "Dockerfile"
 	terraformDeployerRun        = "run.sh"
 	terraformOutputPrefix       = "TF_OUTPUT_"
 	terraformOutputJsonFile     = "tfOutputValues.json"
 )
 
+//go:embed _static/localstack_deployer.yml
+var localstackDeployerYmlContent string
+
 //go:embed _static/terraform_deployer.yml
 var terraformDeployerYmlContent string
 
@@ -97,12 +101,21 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic
 	}
 
 	ymlPaths := []string{filepath.Join(configDir, terraformDeployerYml)}
+
+	localstackYmlPath := filepath.Join(configDir, localstackDeployerYml)
+	_, err = os.Stat(localstackYmlPath)
+	if err == nil {
+		ymlPaths = append(ymlPaths, localstackYmlPath)
+	}
+
 	envYmlPath := filepath.Join(tsd.definitionsDir, envYmlFile)
 	_, err = os.Stat(envYmlPath)
 	if err == nil {
 		ymlPaths = append(ymlPaths, envYmlPath)
 	}
 
+	logger.Debug("Print the yml Paths %s", ymlPaths)
+
 	tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt)
 
 	service := dockerComposeDeployedService{
@@ -174,6 +187,11 @@ func (tsd TerraformServiceDeployer) installDockerfile() (string, error) {
 	tfDir := filepath.Join(locationManager.DeployerDir(), terraformDeployerDir)
 
 	resources := []resource.Resource{
+		&resource.File{
+			Path:         localstackDeployerYml,
+			Content:      resource.FileContentLiteral(localstackDeployerYmlContent),
+			CreateParent: true,
+		},
 		&resource.File{
 			Path:         terraformDeployerYml,
 			Content:      resource.FileContentLiteral(terraformDeployerYmlContent),

test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/cloudwatch.tf

@@ -0,0 +1,173 @@
+resource "aws_cloudwatch_metric_stream" "main" {
+  name          = "my-metric-stream"
+  role_arn      = aws_iam_role.metric_stream_to_firehose.arn
+  firehose_arn  = aws_kinesis_firehose_delivery_stream.s3_stream.arn
+  output_format = "json"
+
+  include_filter {
+    namespace    = "AWS/EC2"
+    metric_names = ["CPUUtilization", "NetworkOut"]
+  }
+
+  include_filter {
+    namespace    = "AWS/EBS"
+    metric_names = []
+  }
+}
+
+# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-trustpolicy.html
+data "aws_iam_policy_document" "streams_assume_role" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["streams.metrics.cloudwatch.amazonaws.com"]
+    }
+
+    actions = [
+      "sts:AssumeRole",
+      "iam:passRole",
+      "cloudwatch:PutMetricStream"
+    ]
+  }
+}
+
+resource "aws_iam_role" "metric_stream_to_firehose" {
+  name               = "metric_stream_to_firehose_role"
+  assume_role_policy = data.aws_iam_policy_document.streams_assume_role.json
+}
+
+# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-trustpolicy.html
+data "aws_iam_policy_document" "metric_stream_to_firehose" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "firehose:PutRecord",
+      "firehose:PutRecordBatch",
+    ]
+
+    resources = [aws_kinesis_firehose_delivery_stream.s3_stream.arn]
+  }
+}
+resource "aws_iam_role_policy" "metric_stream_to_firehose" {
+  name   = "default"
+  role   = aws_iam_role.metric_stream_to_firehose.id
+  policy = data.aws_iam_policy_document.metric_stream_to_firehose.json
+}
+
+resource "aws_s3_bucket" "bucket" {
+  bucket = "metric-stream-test-bucket"
+}
+
+resource "aws_s3_bucket_acl" "bucket_acl" {
+  bucket = aws_s3_bucket.bucket.id
+  acl    = "private"
+}
+
+data "aws_iam_policy_document" "firehose_assume_role" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com"]
+    }
+
+    actions = [
+      "sts:AssumeRole",
+      "iam:passRole",
+      "cloudwatch:PutMetricStream"
+    ]
+  }
+}
+
+resource "aws_iam_role" "firehose_to_s3" {
+  assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json
+}
+
+data "aws_iam_policy_document" "firehose_to_s3" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:AbortMultipartUpload",
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:PutObject",
+    ]
+
+    resources = [
+      aws_s3_bucket.bucket.arn,
+      "${aws_s3_bucket.bucket.arn}/*",
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "firehose_to_s3" {
+  name   = "default"
+  role   = aws_iam_role.firehose_to_s3.id
+  policy = data.aws_iam_policy_document.firehose_to_s3.json
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "s3_stream" {
+  name        = "metric-stream-test-stream"
+  destination = "s3"
+
+  s3_configuration {
+    role_arn   = aws_iam_role.firehose_to_s3.arn
+    bucket_arn = aws_s3_bucket.bucket.arn
+  }
+}
+
+resource "aws_iam_user" "ecs_deployer" {
+  name = "ecs_deployer"
+  path = "*"
+}
+
+# The most important part is the iam:PassRole. With that, this user can give roles to ECS tasks.
+# In theory the user can give the task Admin rights. To make sure that does not happen we restrict
+# the user and allow him only to hand out roles in /ecs/ path. You still need to be careful not
+# to have any roles in there with full admin rights, but no ECS task should have these rights!
+resource "aws_iam_user_policy" "ecs_deployer_policy" {
+  name = "ecs_deployer_policy"
+  user = aws_iam_user.ecs_deployer.name
+  policy = jsonencode(
+    {
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Effect" : "Allow",
+          "Action" : [
+            "ecs:RegisterTaskDefinition",
+            "ecs:DescribeTaskDefinitions",
+            "ecs:ListTaskDefinitions",
+            "ecs:CreateService",
+            "ecs:UpdateService",
+            "ecs:DescribeServices",
+            "ecs:ListServices"
+          ],
+          "Resource" : "*"
+        },
+        {
+          "Effect" : "Allow",
+          "Action" : [
+            "cloudwatch:PutMetricStream"
+          ],
+          "Resource" : "*"
+        },
+        {
+          "Effect" : "Allow",
+          "Action" : ["iam:PassRole"],
+          "Resource" : "*"
+        }
+      ]
+  })
+}
+
+resource "aws_iam_access_key" "ecs_deployer" {
+  user = aws_iam_user.ecs_deployer.name
+}

test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml

@@ -2,8 +2,4 @@ version: '2.3'
 services:
   terraform:
     environment:
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
-      - AWS_PROFILE=${AWS_PROFILE}
       - AWS_REGION=${AWS_REGION:-us-east-1}

test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf

@@ -3,35 +3,37 @@ variable "TEST_RUN_ID" {
 }
 
 provider "aws" {
-  default_tags {
-    tags = {
-      environment  = var.ENVIRONMENT
-      repo         = var.REPO
-      branch       = var.BRANCH
-      build        = var.BUILD_ID
-      created_date = var.CREATED_DATE
-    }
-  }
-}
+  access_key                  = "test"
+  secret_key                  = "test"
+  region                      = "us-east-1"
+  s3_use_path_style           = true
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
 
-resource "aws_instance" "i" {
-  ami           = data.aws_ami.latest-amzn.id
-  monitoring = true
-  instance_type = "t1.micro"
-  tags = {
-    Name = "elastic-package-test-${var.TEST_RUN_ID}"
+  endpoints {
+    apigateway     = "http://localstack:4566"
+    apigatewayv2   = "http://localstack:4566"
+    cloudformation = "http://localstack:4566"
+    cloudwatch     = "http://localstack:4566"
+    dynamodb       = "http://localstack:4566"
+    ec2            = "http://localstack:4566"
+    es             = "http://localstack:4566"
+    elasticache    = "http://localstack:4566"
+    firehose       = "http://localstack:4566"
+    iam            = "http://localstack:4566"
+    kinesis        = "http://localstack:4566"
+    lambda         = "http://localstack:4566"
+    rds            = "http://localstack:4566"
+    redshift       = "http://localstack:4566"
+    route53        = "http://localstack:4566"
+    s3             = "http://localstack:4566"
+    secretsmanager = "http://localstack:4566"
+    ses            = "http://localstack:4566"
+    sns            = "http://localstack:4566"
+    sqs            = "http://localstack:4566"
+    ssm            = "http://localstack:4566"
+    stepfunctions  = "http://localstack:4566"
+    sts            = "http://localstack:4566"
   }
-}
-
-data "aws_ami" "latest-amzn" {
-  most_recent = true
-  owners = [ "amazon" ] # AWS
-  filter {
-    name   = "name"
-    values = ["amzn2-ami-minimal-hvm-*-ebs"]
-  }
-}
-
-output "instance_id" {
-  value = aws_instance.i.id
 }