diff --git a/.github/workflows/integration-test-itar.yaml b/.github/workflows/integration-test-itar.yaml new file mode 100644 index 0000000000..94b70aa53d --- /dev/null +++ b/.github/workflows/integration-test-itar.yaml @@ -0,0 +1,202 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +name: Run Integration Tests In ITAR +env: + PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_ITAR }} + TERRAFORM_AWS_ASSUME_ROLE: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} + TERRAFORM_AWS_ASSUME_ROLE_DURATION: 14400 # 4 hours + S3_INTEGRATION_BUCKET: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} + KEY_NAME: ${{ vars.KEY_NAME_ITAR }} + CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test" + CWA_GITHUB_TEST_REPO_URL: "https://github.com/aws/amazon-cloudwatch-agent-test.git" + CWA_GITHUB_TEST_REPO_BRANCH: "main" + +on: + push: + branches: + - main* + paths-ignore: + - '**/*.md' + - 'NOTICE' + - 'RELEASE_NOTES' + - 'THIRD-PARTY' + - 'LICENSE' + - '.github/**' + - '!.github/workflows/integration-test.yml' + workflow_dispatch: + inputs: + plugins: + description: 'Comma delimited list of plugins to test. Default is empty, and tests everything' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + BuildAndUpload: + uses: ./.github/workflows/test-build.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + BucketKey: "integration-test/binary/${{ github.sha }}" + PackageBucketKey: "integration-test/packaging/${{ github.sha }}" + + GenerateTestMatrix: + name: 'GenerateTestMatrix' + runs-on: ubuntu-latest + outputs: + ec2_linux_matrix: ${{ steps.set-matrix.outputs.ec2_linux_matrix }} + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Set up Go 1.x + uses: actions/setup-go@v4 + with: + go-version: ~1.21.1 + + - name: Generate matrix + id: set-matrix + run: | + go run --tags=generator generator/test_case_generator.go + echo "::set-output name=ec2_linux_matrix::$(echo $(cat generator/resources/ec2_linux_complete_test_matrix.json))" + + - name: Echo test plan matrix + run: | + echo "ec2_linux_matrix: ${{ steps.set-matrix.outputs.ec2_linux_matrix }}" + + OutputEnvVariables: + name: 'OutputEnvVariables' + runs-on: ubuntu-latest + outputs: + CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }} + CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }} + CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Set up Go 1.x + uses: actions/setup-go@v4 + with: + go-version: ~1.21.1 + + - name: SetOutputs + id: set-outputs + run: | + echo "::set-output name=CWA_GITHUB_TEST_REPO_NAME::${{ env.CWA_GITHUB_TEST_REPO_NAME }}" + echo "::set-output name=CWA_GITHUB_TEST_REPO_URL::${{ env.CWA_GITHUB_TEST_REPO_URL }}" + echo "::set-output name=CWA_GITHUB_TEST_REPO_BRANCH::${{ env.CWA_GITHUB_TEST_REPO_BRANCH }}" + + - name: Echo test variables + run: | + echo "CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}" + echo "CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}" + echo "CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}" + + StartLocalStack: + name: 'StartLocalStack' + runs-on: ubuntu-latest + defaults: + run: + working-directory: terraform/ec2/localstack + outputs: + local_stack_host_name: ${{ steps.localstack.outputs.local_stack_host_name }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Verify Terraform version + run: terraform --version + + - name: Terraform init + run: terraform init + + - name: Terraform apply + id: localstack + run: > + echo run terraform and execute test code && + terraform apply --auto-approve + -var="ssh_key_value=${PRIVATE_KEY}" + -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" + -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" + -var="cwa_github_sha=${GITHUB_SHA}" + -var="s3_bucket=${S3_INTEGRATION_BUCKET}" + -var="ssh_key_name=${KEY_NAME}" && + LOCAL_STACK_HOST_NAME=$(terraform output -raw public_dns) && + echo $LOCAL_STACK_HOST_NAME && + echo "::set-output name=local_stack_host_name::$LOCAL_STACK_HOST_NAME" && + aws s3 cp terraform.tfstate s3://${S3_INTEGRATION_BUCKET}/integration-test/local-stack-terraform-state/${GITHUB_SHA}/terraform.tfstate + + EC2LinuxIntegrationTest: + needs: [ BuildAndUpload, StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] + name: 'EC2Linux' + uses: ./.github/workflows/ec2-integration-test.yml + with: + github_sha: ${{github.sha}} + test_dir: terraform/ec2/linux + job_id: ec2-linux-integration-test + test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_matrix}} + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} + secrets: inherit + + StopLocalStack: + name: 'StopLocalStack' + runs-on: ubuntu-latest + if: ${{ always() }} + needs: [ StartLocalStack, EC2LinuxIntegrationTest, LinuxOnPremIntegrationTest ] + defaults: + run: + working-directory: terraform/ec2/localstack + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Copy state + run: aws s3 cp s3://${S3_INTEGRATION_BUCKET}/integration-test/local-stack-terraform-state/${GITHUB_SHA}/terraform.tfstate . + + - name: Verify Terraform version + run: terraform --version + + - name: Terraform init + run: terraform init + + - name: Terraform destroy + run: terraform destroy --auto-approve \ No newline at end of file diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 8962a9509f..e62da09380 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -47,10 +47,19 @@ jobs: id-token: write contents: read with: - ContainerRepositoryNameAndTag: "cwagent-integration-test:${{ github.sha }}" BucketKey: "integration-test/binary/${{ github.sha }}" PackageBucketKey: "integration-test/packaging/${{ github.sha }}" + BuildDocker: + uses: ./.github/workflows/test-build-docker.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + ContainerRepositoryNameAndTag: "cwagent-integration-test:${{ github.sha }}" + BucketKey: "integration-test/binary/${{ github.sha }}" + GenerateTestMatrix: name: 'GenerateTestMatrix' runs-on: ubuntu-latest @@ -575,7 +584,7 @@ jobs: ECSEC2IntegrationTest: name: 'ECSEC2IntegrationTest' runs-on: ubuntu-latest - needs: [ BuildAndUpload, GenerateTestMatrix ] + needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] strategy: fail-fast: false matrix: @@ -658,7 +667,7 @@ jobs: ECSFargateIntegrationTest: name: 'ECSFargateIntegrationTest' runs-on: ubuntu-latest - needs: [BuildAndUpload, GenerateTestMatrix] + needs: [BuildAndUpload, BuildDocker, GenerateTestMatrix] strategy: fail-fast: false matrix: @@ -736,7 +745,7 @@ jobs: EKSIntegrationTest: name: 'EKSIntegrationTest' runs-on: ubuntu-latest - needs: [ BuildAndUpload, GenerateTestMatrix ] + needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] strategy: fail-fast: false matrix: @@ -818,7 +827,7 @@ jobs: EKSPrometheusIntegrationTest: name: 'EKSPrometheusIntegrationTest' runs-on: ubuntu-latest - needs: [ BuildAndUpload, GenerateTestMatrix ] + needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] strategy: fail-fast: false matrix: @@ -1174,7 +1183,7 @@ jobs: EKSEndToEndTest: name: "AppSignals E2E EKS Test" - needs: [ BuildAndUpload ] + needs: [ BuildAndUpload, BuildDocker ] uses: ./.github/workflows/appsignals-e2e-eks-test.yml permissions: id-token: write @@ -1185,7 +1194,7 @@ jobs: EC2EndToEndTest: name: "AppSignals E2E EC2 Test" - needs: [ BuildAndUpload ] + needs: [ BuildAndUpload, BuildDocker ] uses: ./.github/workflows/appsignals-e2e-ec2-test.yml permissions: id-token: write diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index d592b09681..ae06078b41 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -13,6 +13,15 @@ jobs: id-token: write contents: read with: - ContainerRepositoryNameAndTag: "nightly-build:latest" BucketKey: "nightly-build/latest" PackageBucketKey: "nightly-build/latest" + + BuildDocker: + uses: ./.github/workflows/test-build-docker.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + ContainerRepositoryNameAndTag: "nightly-build:latest" + BucketKey: "nightly-build/latest" diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml index a6ddfdc557..7b3876a570 100644 --- a/.github/workflows/soak-test.yml +++ b/.github/workflows/soak-test.yml @@ -30,13 +30,22 @@ jobs: id-token: write contents: read with: - ContainerRepositoryNameAndTag: "cwagent-integration-test:${{ github.sha }}" BucketKey: "integration-test/binary/${{ github.sha }}" PackageBucketKey: "integration-test/binary/${{ github.sha }}" + BuildDocker: + uses: ./.github/workflows/test-build-docker.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + ContainerRepositoryNameAndTag: "cwagent-integration-test:${{ github.sha }}" + BucketKey: "integration-test/binary/${{ github.sha }}" + DeploySoakTest: name: "DeploySoakTest" - needs: [BuildAndUpload] + needs: [BuildAndUpload, BuildDocker] runs-on: ubuntu-latest permissions: id-token: write diff --git a/.github/workflows/test-build-docker.yaml b/.github/workflows/test-build-docker.yaml new file mode 100644 index 0000000000..0d53e24372 --- /dev/null +++ b/.github/workflows/test-build-docker.yaml @@ -0,0 +1,94 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +name: Build And Upload Docker Image +env: + CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test" + +on: + workflow_dispatch: + inputs: + ContainerRepositoryNameAndTag: + # e.g. "cwagent-integration-test:SHA" + # e.g. "cwa-release:latest" + # e.g. "cwa_nonprod:latest" + description: "ECR repo name and tag" + required: true + type: string + BucketKey: + # e.g. s3:///integration-test/binary/" + # e.g. s3:///nonprod + # e.g. s3:///release + description: "S3 URI to upload artifacts into." + required: true + type: string + workflow_call: + inputs: + ContainerRepositoryNameAndTag: + # e.g. "cwagent-integration-test:SHA" + # e.g. "cwa-release:latest" + # e.g. "cwa_nonprod:latest" + description: "ECR repo name and tag" + required: true + type: string + BucketKey: + # e.g. s3:///integration-test/binary/" + # e.g. s3:///nonprod + # e.g. s3:///release + description: "S3 URI to upload artifacts into." + required: true + type: string + +jobs: + MakeBinary: + name: 'MakeBinary' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + + - name: Cache container + id: cached_container + uses: actions/cache@v3 + with: + key: "cached_container_${{ github.sha }}" + path: go.mod + + - name: Login ECR + if: contains(inputs.BucketKey, 'test') == false || steps.cached_container.outputs.cache-hit == false + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Set up Docker Buildx + if: contains(inputs.BucketKey, 'test') == false || steps.cached_container.outputs.cache-hit == false + uses: docker/setup-buildx-action@v1 + + - name: Set up QEMU + if: contains(inputs.BucketKey, 'test') == false || steps.cached_container.outputs.cache-hit == false + uses: docker/setup-qemu-action@v1 + + # Build dir is ignored in our .dockerignore thus need to copy to another dir. + - name: Copy Binary For Agent Image Build + if: contains(inputs.BucketKey, 'test') == false || steps.cached_container.outputs.cache-hit == false + run: aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }} . --recursive + + - name: Build Cloudwatch Agent Image + uses: docker/build-push-action@v4 + if: contains(inputs.BucketKey, 'test') == false || steps.cached_container.outputs.cache-hit == false + with: + file: amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/localdeb/Dockerfile + context: . + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ inputs.ContainerRepositoryNameAndTag }} + platforms: linux/amd64, linux/arm64 \ No newline at end of file diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index def7f549b8..94e1a78826 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -8,13 +8,6 @@ env: on: workflow_dispatch: inputs: - ContainerRepositoryNameAndTag: - # e.g. "cwagent-integration-test:SHA" - # e.g. "cwa-release:latest" - # e.g. "cwa_nonprod:latest" - description: "ECR repo name and tag" - required: true - type: string BucketKey: # e.g. s3:///integration-test/binary/" # e.g. s3:///nonprod @@ -28,13 +21,6 @@ on: type: string workflow_call: inputs: - ContainerRepositoryNameAndTag: - # e.g. "cwagent-integration-test:SHA" - # e.g. "cwa-release:latest" - # e.g. "cwa_nonprod:latest" - description: "ECR repo name and tag" - required: true - type: string BucketKey: # e.g. s3:///integration-test/binary/" # e.g. s3:///nonprod @@ -117,35 +103,6 @@ jobs: aws s3 cp build/bin/linux/amd64/amazon-cloudwatch-agent.rpm s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm aws s3 cp build/bin/linux/arm64/amazon-cloudwatch-agent.rpm s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.BucketKey }}/amazon_linux/arm64/latest/amazon-cloudwatch-agent.rpm - - name: Login ECR - if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Set up Docker Buildx - if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false - uses: docker/setup-buildx-action@v1 - - - name: Set up QEMU - if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false - uses: docker/setup-qemu-action@v1 - - # Build dir is ignored in our .dockerignore thus need to copy to another dir. - - name: Copy Binary For Agent Image Build - if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false - run: cp -r build/bin/linux/* . - - - name: Build Cloudwatch Agent Image - uses: docker/build-push-action@v4 - if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false - with: - file: amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/localdeb/Dockerfile - context: . - push: true - tags: | - ${{ steps.login-ecr.outputs.registry }}/${{ inputs.ContainerRepositoryNameAndTag }} - platforms: linux/amd64, linux/arm64 - MakeMSIZip: name: 'MakeMSIZip' runs-on: ubuntu-latest