From d96ed6f5acb4a719dd0851b2c96e85c63643adf7 Mon Sep 17 00:00:00 2001 From: lisguo Date: Thu, 1 Aug 2024 10:26:40 -0400 Subject: [PATCH] uncomment some tests to see if there's an issue with terraform --- .github/workflows/integration-test.yml | 1922 ++++++++++++------------ 1 file changed, 961 insertions(+), 961 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index c1f809a1f1..b012632cc8 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -68,44 +68,44 @@ jobs: PackageBucketKey: "integration-test/packaging/${{ github.sha }}" TerraformAWSAssumeRole: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} Bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - # - # BuildDocker: - # needs: [BuildAndUpload] - # uses: ./.github/workflows/test-build-docker.yml - # secrets: inherit - # permissions: - # id-token: write - # contents: read - # with: - # ContainerRepositoryNameAndTag: "cwagent-integration-test:${{ github.sha }}" - # BucketKey: "integration-test/binary/${{ github.sha }}" - # PackageBucketKey: "integration-test/packaging/${{ github.sha }}" - # - # BuildAndUploadITAR: - # uses: ./.github/workflows/test-build.yml - # secrets: inherit - # permissions: - # id-token: write - # contents: read - # with: - # BucketKey: "integration-test/binary/${{ github.sha }}" - # PackageBucketKey: "integration-test/packaging/${{ github.sha }}" - # Region: "us-gov-east-1" - # TerraformAWSAssumeRole: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - # Bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} - # - # BuildAndUploadCN: - # uses: ./.github/workflows/test-build.yml - # secrets: inherit - # permissions: - # id-token: write - # contents: read - # with: - # BucketKey: "integration-test/binary/${{ github.sha }}" - # PackageBucketKey: "integration-test/packaging/${{ github.sha }}" - # Region: "cn-north-1" - # TerraformAWSAssumeRole: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - # Bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} + + BuildDocker: + needs: [BuildAndUpload] + uses: ./.github/workflows/test-build-docker.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + ContainerRepositoryNameAndTag: "cwagent-integration-test:${{ github.sha }}" + BucketKey: "integration-test/binary/${{ github.sha }}" + PackageBucketKey: "integration-test/packaging/${{ github.sha }}" + + BuildAndUploadITAR: + uses: ./.github/workflows/test-build.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + BucketKey: "integration-test/binary/${{ github.sha }}" + PackageBucketKey: "integration-test/packaging/${{ github.sha }}" + Region: "us-gov-east-1" + TerraformAWSAssumeRole: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} + Bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} + + BuildAndUploadCN: + uses: ./.github/workflows/test-build.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + BucketKey: "integration-test/binary/${{ github.sha }}" + PackageBucketKey: "integration-test/packaging/${{ github.sha }}" + Region: "cn-north-1" + TerraformAWSAssumeRole: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} + Bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} GenerateTestMatrix: name: 'GenerateTestMatrix' @@ -271,121 +271,121 @@ jobs: - # EC2NvidiaGPUIntegrationTest: - # needs: [ BuildAndUpload, StartLocalStack, GenerateTestMatrix ] - # name: 'EC2NVIDIAGPUIntegrationTest' - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_gpu_matrix) }} - # permissions: - # id-token: write - # contents: read - # steps: - # - uses: actions/checkout@v3 - # with: - # repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - # ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - # - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - # aws-region: us-west-2 - # role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - # - # - name: Cache if success - # id: ec2-linux-integration-test - # uses: actions/cache@v3 - # with: - # path: go.mod - # key: ec2-nvidia-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - # - # - name: Echo Test Info - # run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} - # - # - name: Verify Terraform version - # run: terraform --version - # - # # nick-fields/retry@v2 starts at base dir - # - name: Terraform apply - # if: ${{ matrix.arrays.family == 'linux' && steps.ec2-nvidia-integration-test.outputs.cache-hit != 'true' }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 30 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/ec2/linux - # fi - # - # terraform init - # if terraform apply --auto-approve \ - # -var="ssh_key_value=${PRIVATE_KEY}" -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - # -var="test_name=${{ matrix.arrays.os }}" \ - # -var="cwa_github_sha=${GITHUB_SHA}" -var="install_agent=${{ matrix.arrays.installAgentCommand }}" \ - # -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" \ - # -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - # -var="user=${{ matrix.arrays.username }}" \ - # -var="ami=${{ matrix.arrays.ami }}" \ - # -var="ca_cert_path=${{ matrix.arrays.caCertPath }}" \ - # -var="arc=${{ matrix.arrays.arc }}" \ - # -var="binary_name=${{ matrix.arrays.binaryName }}" \ - # -var="local_stack_host_name=${{ needs.StartLocalStack.outputs.local_stack_host_name }}" \ - # -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - # -var="ssh_key_name=${KEY_NAME}" \ - # -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # - name: Terraform apply - # if: ${{ matrix.arrays.family == 'window' && steps.ec2-nvidia-integration-test.outputs.cache-hit != 'true' }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 30 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/ec2/win - # fi - # terraform init - # if terraform apply --auto-approve \ - # -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ - # -var="github_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - # -var="cwa_github_sha=${GITHUB_SHA}" -var="ami=${{ matrix.arrays.ami }}" \ - # -var="test_dir=${{ matrix.arrays.test_dir }}" \ - # -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - # -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - # -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" \ - # -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # #This is here just in case workflow cancel - # - name: Terraform destroy - # if: ${{ cancelled() || failure() }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 8 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # elif if "${{ matrix.arrays.os }}" == window; then - # cd terraform/ec2/win - # else - # cd terraform/ec2/linux - # fi - # terraform destroy --auto-approve - # + EC2NvidiaGPUIntegrationTest: + needs: [ BuildAndUpload, StartLocalStack, GenerateTestMatrix ] + name: 'EC2NVIDIAGPUIntegrationTest' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_gpu_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Cache if success + id: ec2-linux-integration-test + uses: actions/cache@v3 + with: + path: go.mod + key: ec2-nvidia-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} + + - name: Echo Test Info + run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} + + - name: Verify Terraform version + run: terraform --version + + # nick-fields/retry@v2 starts at base dir + - name: Terraform apply + if: ${{ matrix.arrays.family == 'linux' && steps.ec2-nvidia-integration-test.outputs.cache-hit != 'true' }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 30 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/ec2/linux + fi + + terraform init + if terraform apply --auto-approve \ + -var="ssh_key_value=${PRIVATE_KEY}" -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ + -var="test_name=${{ matrix.arrays.os }}" \ + -var="cwa_github_sha=${GITHUB_SHA}" -var="install_agent=${{ matrix.arrays.installAgentCommand }}" \ + -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" \ + -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ + -var="user=${{ matrix.arrays.username }}" \ + -var="ami=${{ matrix.arrays.ami }}" \ + -var="ca_cert_path=${{ matrix.arrays.caCertPath }}" \ + -var="arc=${{ matrix.arrays.arc }}" \ + -var="binary_name=${{ matrix.arrays.binaryName }}" \ + -var="local_stack_host_name=${{ needs.StartLocalStack.outputs.local_stack_host_name }}" \ + -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ + -var="ssh_key_name=${KEY_NAME}" \ + -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + - name: Terraform apply + if: ${{ matrix.arrays.family == 'window' && steps.ec2-nvidia-integration-test.outputs.cache-hit != 'true' }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 30 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/ec2/win + fi + terraform init + if terraform apply --auto-approve \ + -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ + -var="github_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ + -var="cwa_github_sha=${GITHUB_SHA}" -var="ami=${{ matrix.arrays.ami }}" \ + -var="test_dir=${{ matrix.arrays.test_dir }}" \ + -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ + -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ + -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" \ + -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + #This is here just in case workflow cancel + - name: Terraform destroy + if: ${{ cancelled() || failure() }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + elif if "${{ matrix.arrays.os }}" == window; then + cd terraform/ec2/win + else + cd terraform/ec2/linux + fi + terraform destroy --auto-approve + OutputEnvVariables: name: 'OutputEnvVariables' runs-on: ubuntu-latest @@ -416,77 +416,77 @@ jobs: echo "CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}" echo "CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}" echo "CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}" - # - # EC2LinuxIntegrationTest: - # needs: [ BuildAndUpload, StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] - # name: 'EC2Linux' - # uses: ./.github/workflows/ec2-integration-test.yml - # with: - # github_sha: ${{github.sha}} - # test_dir: terraform/ec2/linux - # job_id: ec2-linux-integration-test - # test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_matrix}} - # test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - # test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - # test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - # localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} - # region: us-west-2 - # terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} - # s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - # secrets: inherit - # - # EC2LinuxIntegrationTestITAR: - # needs: [ BuildAndUpload, BuildAndUploadITAR, StartLocalStackITAR, GenerateTestMatrix, OutputEnvVariables ] - # name: 'EC2LinuxITAR' - # uses: ./.github/workflows/ec2-integration-test.yml - # with: - # github_sha: ${{github.sha}} - # test_dir: terraform/ec2/linux - # job_id: ec2-linux-integration-test - # test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_itar_matrix}} - # test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - # test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - # test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - # localstack_host: ${{needs.StartLocalStackITAR.outputs.local_stack_host_name}} - # region: us-gov-east-1 - # terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - # s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} - # secrets: inherit - # - # EC2LinuxIntegrationTestCN: - # needs: [ BuildAndUpload, BuildAndUploadCN, StartLocalStackCN, GenerateTestMatrix, OutputEnvVariables ] - # name: 'EC2LinuxCN' - # uses: ./.github/workflows/ec2-integration-test.yml - # with: - # github_sha: ${{github.sha}} - # test_dir: terraform/ec2/linux - # job_id: ec2-linux-integration-test - # test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_china_matrix}} - # test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - # test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - # test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - # localstack_host: ${{needs.StartLocalStackCN.outputs.local_stack_host_name}} - # region: cn-north-1 - # terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - # s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - # secrets: inherit - # - # - # LinuxOnPremIntegrationTest: - # needs: [BuildAndUpload, StartLocalStack, GenerateTestMatrix, OutputEnvVariables] - # name: 'OnpremLinux' - # uses: ./.github/workflows/ec2-integration-test.yml - # with: - # github_sha: ${{github.sha}} - # test_dir: terraform/ec2/linux_onprem - # job_id: linux-onprem-integration-test - # test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_onprem_matrix}} - # test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - # test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - # test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - # localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} - # region: us-west-2 - # secrets: inherit + + EC2LinuxIntegrationTest: + needs: [ BuildAndUpload, StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] + name: 'EC2Linux' + uses: ./.github/workflows/ec2-integration-test.yml + with: + github_sha: ${{github.sha}} + test_dir: terraform/ec2/linux + job_id: ec2-linux-integration-test + test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_matrix}} + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} + region: us-west-2 + terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} + s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} + secrets: inherit + + EC2LinuxIntegrationTestITAR: + needs: [ BuildAndUpload, BuildAndUploadITAR, StartLocalStackITAR, GenerateTestMatrix, OutputEnvVariables ] + name: 'EC2LinuxITAR' + uses: ./.github/workflows/ec2-integration-test.yml + with: + github_sha: ${{github.sha}} + test_dir: terraform/ec2/linux + job_id: ec2-linux-integration-test + test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_itar_matrix}} + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + localstack_host: ${{needs.StartLocalStackITAR.outputs.local_stack_host_name}} + region: us-gov-east-1 + terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} + s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} + secrets: inherit + + EC2LinuxIntegrationTestCN: + needs: [ BuildAndUpload, BuildAndUploadCN, StartLocalStackCN, GenerateTestMatrix, OutputEnvVariables ] + name: 'EC2LinuxCN' + uses: ./.github/workflows/ec2-integration-test.yml + with: + github_sha: ${{github.sha}} + test_dir: terraform/ec2/linux + job_id: ec2-linux-integration-test + test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_china_matrix}} + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + localstack_host: ${{needs.StartLocalStackCN.outputs.local_stack_host_name}} + region: cn-north-1 + terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} + s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} + secrets: inherit + + + LinuxOnPremIntegrationTest: + needs: [BuildAndUpload, StartLocalStack, GenerateTestMatrix, OutputEnvVariables] + name: 'OnpremLinux' + uses: ./.github/workflows/ec2-integration-test.yml + with: + github_sha: ${{github.sha}} + test_dir: terraform/ec2/linux_onprem + job_id: linux-onprem-integration-test + test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_onprem_matrix}} + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} + region: us-west-2 + secrets: inherit EC2WinIntegrationTest: needs: [BuildAndUpload, BuildAndUploadPackages, GenerateTestMatrix] @@ -570,537 +570,14 @@ jobs: fi terraform destroy --auto-approve - # EC2DarwinIntegrationTest: - # needs: [BuildAndUpload, BuildAndUploadPackages, GenerateTestMatrix] - # name: 'EC2DarwinIntegrationTest' - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_mac_matrix) }} - # permissions: - # id-token: write - # contents: read - # steps: - # - uses: actions/checkout@v3 - # with: - # repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - # ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - # - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - # aws-region: us-west-2 - # role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - # - # - name: Cache if success - # id: ec2-mac-integration-test - # uses: actions/cache@v3 - # with: - # path: go.mod - # key: ec2-mac-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - # - # - name: Echo OS - # run: echo run on ec2 instance os ${{ matrix.arrays.os }} - # - # - name: Verify Terraform version - # run: terraform --version - # - # # nick-fields/retry@v2 starts at base dir - # - name: Terraform apply - # if: steps.ec2-mac-integration-test.outputs.cache-hit != 'true' - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 30 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/ec2/mac - # fi - # terraform init - # if terraform apply --auto-approve \ - # -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ - # -var="arc=${{ matrix.arrays.arc }}" \ - # -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - # -var="cwa_github_sha=${GITHUB_SHA}" -var="ami=${{ matrix.arrays.ami }}" \ - # -var="test_dir=${{ matrix.arrays.test_dir }}" \ - # -var="license_manager_arn=${{ env.LICENSE_MANAGER_ARN }}" \ - # -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then - # terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # - # #This is here just in case workflow cancel - # - name: Terraform destroy - # if: ${{ cancelled() || failure() }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 8 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/ec2/mac - # fi - # terraform destroy --auto-approve - - # StopLocalStack: - # name: 'StopLocalStack' - # if: ${{ always() }} - # needs: [ StartLocalStack, EC2LinuxIntegrationTest, LinuxOnPremIntegrationTest, OutputEnvVariables ] - # uses: ./.github/workflows/stop-localstack.yml - # secrets: inherit - # permissions: - # id-token: write - # contents: read - # with: - # region: us-west-2 - # test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - # test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - # terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} - # github_sha: ${{github.sha}} - # s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - - # StopLocalStackITAR: - # name: 'StopLocalStackITAR' - # if: ${{ always() }} - # needs: [ EC2LinuxIntegrationTestITAR, OutputEnvVariables ] - # uses: ./.github/workflows/stop-localstack.yml - # secrets: inherit - # permissions: - # id-token: write - # contents: read - # with: - # region: us-gov-east-1 - # test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - # test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - # terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - # github_sha: ${{github.sha}} - # s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} - # - # StopLocalStackCN: - # name: 'StopLocalStackCN' - # if: ${{ always() }} - # needs: [ EC2LinuxIntegrationTestCN ] - # uses: ./.github/workflows/stop-localstack.yml - # secrets: inherit - # permissions: - # id-token: write - # contents: read - # with: - # region: cn-north-1 - # test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - # test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - # terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - # github_sha: ${{github.sha}} - # s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - # - # ECSEC2IntegrationTest: - # name: 'ECSEC2IntegrationTest' - # runs-on: ubuntu-latest - # needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] - # strategy: - # fail-fast: false - # matrix: - # arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_ec2_launch_daemon_matrix) }} - # permissions: - # id-token: write - # contents: read - # steps: - # - uses: actions/checkout@v3 - # with: - # repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - # ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - # - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - # aws-region: us-west-2 - # role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - # - # - name: Cache if success - # id: ecs-ec2-integration-test - # uses: actions/cache@v3 - # with: - # path: go.mod - # key: ecs-ec2-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} - # - # - name: Login ECR - # id: login-ecr - # if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' - # uses: aws-actions/amazon-ecr-login@v1 - # - # - name: Verify Terraform version - # if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' - # run: terraform --version - # - # - name: Terraform apply - # if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 15 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/ecs_ec2/daemon - # fi - # - # terraform init - # if terraform apply --auto-approve\ - # -var="test_dir=${{ matrix.arrays.test_dir }}"\ - # -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - # -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ - # -var="cwagent_image_tag=${{ github.sha }}"\ - # -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - # -var="metadataEnabled=${{ matrix.arrays.metadataEnabled }}" \ - # -var="ami=${{ matrix.arrays.ami }}" ; then - # terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # - # - name: Terraform destroy - # if: ${{ cancelled() || failure() }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 8 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/ecs_ec2/daemon - # fi - # terraform destroy --auto-approve - # - # ECSFargateIntegrationTest: - # name: 'ECSFargateIntegrationTest' - # runs-on: ubuntu-latest - # needs: [BuildAndUpload, BuildDocker, GenerateTestMatrix] - # strategy: - # fail-fast: false - # matrix: - # arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_fargate_matrix) }} - # permissions: - # id-token: write - # contents: read - # steps: - # - uses: actions/checkout@v3 - # with: - # repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - # ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - # - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - # aws-region: us-west-2 - # role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - # - # - name: Cache if success - # id: ecs-fargate-integration-test - # uses: actions/cache@v3 - # with: - # path: go.mod - # key: ecs-fargate-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} - # - # - name: Login ECR - # id: login-ecr - # if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' - # uses: aws-actions/amazon-ecr-login@v1 - # - # - name: Verify Terraform version - # if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' - # run: terraform --version - # - # - name: Terraform apply - # if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 15 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/ecs_fargate/linux - # fi - # - # terraform init - # if terraform apply --auto-approve\ - # -var="test_dir=${{ matrix.arrays.test_dir }}"\ - # -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ - # -var="cwagent_image_tag=${{ github.sha }}"; then - # terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # - name: Terraform destroy - # if: ${{ cancelled() || failure() }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 8 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/ecs_fargate/linux - # fi - # terraform destroy --auto-approve - # - # EKSIntegrationTest: - # name: 'EKSIntegrationTest' - # runs-on: ubuntu-latest - # needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] - # strategy: - # fail-fast: false - # matrix: - # arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_daemon_matrix) }} - # permissions: - # id-token: write - # contents: read - # steps: - # - uses: actions/checkout@v3 - # with: - # repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - # ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - # - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - # aws-region: us-west-2 - # role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - # - # - name: Cache if success - # id: eks-ec2-integration-test - # uses: actions/cache@v3 - # with: - # path: go.mod - # key: ${{ matrix.arrays.terraform_dir }}-${{ matrix.arrays.k8s_version }}-${{ matrix.arrays.instanceType }}-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} - # - # - name: Login ECR - # id: login-ecr - # if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - # uses: aws-actions/amazon-ecr-login@v1 - # - # - name: Verify Terraform version - # if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - # run: terraform --version - # - # - name: Terraform apply - # if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 2 - # timeout_minutes: 90 # EKS takes about 20 minutes to spin up a cluster and service on the cluster - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/eks/daemon - # fi - # - # terraform init - # if terraform apply --auto-approve \ - # -var="test_dir=${{ matrix.arrays.test_dir }}"\ - # -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ - # -var="cwagent_image_tag=${{ github.sha }}" \ - # -var="ami_type=${{ matrix.arrays.ami }}" \ - # -var="instance_type=${{ matrix.arrays.instanceType }}" \ - # -var="k8s_version=${{ matrix.arrays.k8s_version }}"; then - # terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # - # - name: Terraform destroy - # if: ${{ cancelled() || failure() }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 8 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/eks/daemon - # fi - # terraform destroy --auto-approve - # - # EKSPrometheusIntegrationTest: - # name: 'EKSPrometheusIntegrationTest' - # runs-on: ubuntu-latest - # needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] - # strategy: - # fail-fast: false - # matrix: - # arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_deployment_matrix) }} - # permissions: - # id-token: write - # contents: read - # steps: - # - uses: actions/checkout@v3 - # with: - # repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - # ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - # - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - # aws-region: us-west-2 - # role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - # - # - name: Cache if success - # id: eks-ec2-integration-test - # uses: actions/cache@v3 - # with: - # path: go.mod - # key: eks-ec2-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} - # - # - name: Login ECR - # id: login-ecr - # if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - # uses: aws-actions/amazon-ecr-login@v1 - # - # - name: Verify Terraform version - # if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - # run: terraform --version - # - # - name: Terraform apply - # if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 60 # EKS takes about 20 minutes to spin up a cluster and service on the cluster - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/eks/deployment - # fi - # - # terraform init - # if terraform apply --auto-approve \ - # -var="test_dir=${{ matrix.arrays.test_dir }}"\ - # -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ - # -var="cwagent_image_tag=${{ github.sha }}" \ - # -var="k8s_version=${{ matrix.arrays.k8s_version }}"; then - # terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # - # - name: Terraform destroy - # if: ${{ cancelled() || failure() }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 8 - # retry_wait_seconds: 5 - # command: | - # if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - # cd "${{ matrix.arrays.terraform_dir }}" - # else - # cd terraform/eks/deployment - # fi - # terraform destroy --auto-approve - # - # PerformanceTrackingTest: - # name: "PerformanceTrackingTest" - # needs: [BuildAndUpload, GenerateTestMatrix] - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_performance_matrix) }} - # permissions: - # id-token: write - # contents: read - # steps: - # - uses: actions/checkout@v3 - # with: - # repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - # ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - # - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - # aws-region: us-west-2 - # role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - # - # - name: Cache if success - # id: performance-tracking - # uses: actions/cache@v3 - # with: - # path: go.mod - # key: performance-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - # - # - name: Verify Terraform version - # if: steps.performance-tracking.outputs.cache-hit != 'true' - # run: terraform --version - # - # - name: Terraform apply - # if: steps.performance-tracking.outputs.cache-hit != 'true' - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 1 - # timeout_minutes: 60 - # retry_wait_seconds: 5 - # command: | - # cd terraform/performance - # terraform init - # if terraform apply --auto-approve \ - # -var="ssh_key_value=${PRIVATE_KEY}" \ - # -var="cwa_github_sha=${GITHUB_SHA}" \ - # -var="ami=${{ matrix.arrays.ami }}" \ - # -var="arc=${{ matrix.arrays.arc }}" \ - # -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - # -var="ssh_key_name=${KEY_NAME}" \ - # -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - # -var="family=${{ matrix.arrays.family}}"\ - # -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # - # - name: Terraform destroy - # if: ${{ cancelled() || failure() }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 8 - # retry_wait_seconds: 5 - # command: cd terraform/performance && terraform destroy --auto-approve - - EC2WinPerformanceTest: - name: "EC2WinPerformanceTest" - needs: [ BuildAndUpload, BuildAndUploadPackages, GenerateTestMatrix ] + EC2DarwinIntegrationTest: + needs: [BuildAndUpload, BuildAndUploadPackages, GenerateTestMatrix] + name: 'EC2DarwinIntegrationTest' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_performance_matrix) }} + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_mac_matrix) }} permissions: id-token: write contents: read @@ -1118,40 +595,47 @@ jobs: role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - name: Cache if success - id: performance-tracking + id: ec2-mac-integration-test uses: actions/cache@v3 with: path: go.mod - key: performance-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} + key: ec2-mac-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} + + - name: Echo OS + run: echo run on ec2 instance os ${{ matrix.arrays.os }} - name: Verify Terraform version - if: steps.performance-tracking.outputs.cache-hit != 'true' run: terraform --version + # nick-fields/retry@v2 starts at base dir - name: Terraform apply - if: steps.performance-tracking.outputs.cache-hit != 'true' + if: steps.ec2-mac-integration-test.outputs.cache-hit != 'true' uses: nick-fields/retry@v2 with: - max_attempts: 1 - timeout_minutes: 60 + max_attempts: 3 + timeout_minutes: 30 retry_wait_seconds: 5 command: | - cd terraform/performance + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/ec2/mac + fi terraform init if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${GITHUB_SHA}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="family=${{ matrix.arrays.family}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve + -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ + -var="arc=${{ matrix.arrays.arc }}" \ + -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ + -var="cwa_github_sha=${GITHUB_SHA}" -var="ami=${{ matrix.arrays.ami }}" \ + -var="test_dir=${{ matrix.arrays.test_dir }}" \ + -var="license_manager_arn=${{ env.LICENSE_MANAGER_ARN }}" \ + -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then + terraform destroy -auto-approve else terraform destroy -auto-approve && exit 1 fi + #This is here just in case workflow cancel - name: Terraform destroy if: ${{ cancelled() || failure() }} uses: nick-fields/retry@v2 @@ -1159,86 +643,73 @@ jobs: max_attempts: 3 timeout_minutes: 8 retry_wait_seconds: 5 - command: cd terraform/performance && terraform destroy --auto-approve - # - # StressTrackingTest: - # name: "StressTrackingTest" - # needs: [BuildAndUpload, GenerateTestMatrix] - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_stress_matrix) }} - # permissions: - # id-token: write - # contents: read - # steps: - # - uses: actions/checkout@v3 - # with: - # repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - # ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - # - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - # aws-region: us-west-2 - # role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - # - # - name: Cache if success - # id: stress-tracking - # uses: actions/cache@v3 - # with: - # path: go.mod - # key: stress-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - # - # - name: Verify Terraform version - # if: steps.stress-tracking.outputs.cache-hit != 'true' - # run: terraform --version - # - # - name: Echo Test Info - # run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} - # - # - name: Terraform apply - # if: steps.stress-tracking.outputs.cache-hit != 'true' - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 1 - # timeout_minutes: 60 - # retry_wait_seconds: 5 - # command: | - # cd terraform/stress - # terraform init - # if terraform apply --auto-approve \ - # -var="ssh_key_value=${PRIVATE_KEY}" \ - # -var="cwa_github_sha=${GITHUB_SHA}" \ - # -var="ami=${{ matrix.arrays.ami }}" \ - # -var="arc=${{ matrix.arrays.arc }}" \ - # -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - # -var="ssh_key_name=${KEY_NAME}" \ - # -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - # -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - # else - # terraform destroy -auto-approve && exit 1 - # fi - # - # - name: Terraform destroy - # if: ${{ cancelled() || failure() }} - # uses: nick-fields/retry@v2 - # with: - # max_attempts: 3 - # timeout_minutes: 8 - # retry_wait_seconds: 5 - # command: cd terraform/stress && terraform destroy --auto-approve + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/ec2/mac + fi + terraform destroy --auto-approve - EC2WinStressTrackingTest: - name: "EC2WinStressTrackingTest" - needs: [BuildAndUpload, BuildAndUploadPackages, GenerateTestMatrix] + StopLocalStack: + name: 'StopLocalStack' + if: ${{ always() }} + needs: [ StartLocalStack, EC2LinuxIntegrationTest, LinuxOnPremIntegrationTest, OutputEnvVariables ] + uses: ./.github/workflows/stop-localstack.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + region: us-west-2 + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} + github_sha: ${{github.sha}} + s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} + + StopLocalStackITAR: + name: 'StopLocalStackITAR' + if: ${{ always() }} + needs: [ EC2LinuxIntegrationTestITAR, OutputEnvVariables ] + uses: ./.github/workflows/stop-localstack.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + region: us-gov-east-1 + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} + github_sha: ${{github.sha}} + s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} + + StopLocalStackCN: + name: 'StopLocalStackCN' + if: ${{ always() }} + needs: [ EC2LinuxIntegrationTestCN ] + uses: ./.github/workflows/stop-localstack.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + region: cn-north-1 + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} + github_sha: ${{github.sha}} + s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} + + ECSEC2IntegrationTest: + name: 'ECSEC2IntegrationTest' runs-on: ubuntu-latest + needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] strategy: fail-fast: false matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_stress_matrix) }} + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_ec2_launch_daemon_matrix) }} permissions: id-token: write contents: read @@ -1256,35 +727,564 @@ jobs: role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - name: Cache if success - id: ec2-win-stress-tracking-test + id: ecs-ec2-integration-test uses: actions/cache@v3 with: path: go.mod - key: ec2-win-stress-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} + key: ecs-ec2-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} + + - name: Login ECR + id: login-ecr + if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' + uses: aws-actions/amazon-ecr-login@v1 - name: Verify Terraform version - if: steps.ec2-win-stress-tracking-test.outputs.cache-hit != 'true' + if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' run: terraform --version - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} - - name: Terraform apply - if: steps.ec2-win-stress-tracking-test.outputs.cache-hit != 'true' + if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' uses: nick-fields/retry@v2 with: - max_attempts: 1 - timeout_minutes: 60 + max_attempts: 3 + timeout_minutes: 15 retry_wait_seconds: 5 command: | - cd terraform/stress + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/ecs_ec2/daemon + fi + terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${GITHUB_SHA}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ + if terraform apply --auto-approve\ + -var="test_dir=${{ matrix.arrays.test_dir }}"\ + -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ + -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ + -var="cwagent_image_tag=${{ github.sha }}"\ + -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ + -var="metadataEnabled=${{ matrix.arrays.metadataEnabled }}" \ + -var="ami=${{ matrix.arrays.ami }}" ; then + terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + + - name: Terraform destroy + if: ${{ cancelled() || failure() }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/ecs_ec2/daemon + fi + terraform destroy --auto-approve + + ECSFargateIntegrationTest: + name: 'ECSFargateIntegrationTest' + runs-on: ubuntu-latest + needs: [BuildAndUpload, BuildDocker, GenerateTestMatrix] + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_fargate_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Cache if success + id: ecs-fargate-integration-test + uses: actions/cache@v3 + with: + path: go.mod + key: ecs-fargate-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} + + - name: Login ECR + id: login-ecr + if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' + uses: aws-actions/amazon-ecr-login@v1 + + - name: Verify Terraform version + if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' + run: terraform --version + + - name: Terraform apply + if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 15 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/ecs_fargate/linux + fi + + terraform init + if terraform apply --auto-approve\ + -var="test_dir=${{ matrix.arrays.test_dir }}"\ + -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ + -var="cwagent_image_tag=${{ github.sha }}"; then + terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + - name: Terraform destroy + if: ${{ cancelled() || failure() }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/ecs_fargate/linux + fi + terraform destroy --auto-approve + + EKSIntegrationTest: + name: 'EKSIntegrationTest' + runs-on: ubuntu-latest + needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_daemon_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Cache if success + id: eks-ec2-integration-test + uses: actions/cache@v3 + with: + path: go.mod + key: ${{ matrix.arrays.terraform_dir }}-${{ matrix.arrays.k8s_version }}-${{ matrix.arrays.instanceType }}-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} + + - name: Login ECR + id: login-ecr + if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' + uses: aws-actions/amazon-ecr-login@v1 + + - name: Verify Terraform version + if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' + run: terraform --version + + - name: Terraform apply + if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' + uses: nick-fields/retry@v2 + with: + max_attempts: 2 + timeout_minutes: 90 # EKS takes about 20 minutes to spin up a cluster and service on the cluster + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/eks/daemon + fi + + terraform init + if terraform apply --auto-approve \ + -var="test_dir=${{ matrix.arrays.test_dir }}"\ + -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ + -var="cwagent_image_tag=${{ github.sha }}" \ + -var="ami_type=${{ matrix.arrays.ami }}" \ + -var="instance_type=${{ matrix.arrays.instanceType }}" \ + -var="k8s_version=${{ matrix.arrays.k8s_version }}"; then + terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + + - name: Terraform destroy + if: ${{ cancelled() || failure() }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/eks/daemon + fi + terraform destroy --auto-approve + + EKSPrometheusIntegrationTest: + name: 'EKSPrometheusIntegrationTest' + runs-on: ubuntu-latest + needs: [ BuildAndUpload, BuildDocker, GenerateTestMatrix ] + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_deployment_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Cache if success + id: eks-ec2-integration-test + uses: actions/cache@v3 + with: + path: go.mod + key: eks-ec2-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} + + - name: Login ECR + id: login-ecr + if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' + uses: aws-actions/amazon-ecr-login@v1 + + - name: Verify Terraform version + if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' + run: terraform --version + + - name: Terraform apply + if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 60 # EKS takes about 20 minutes to spin up a cluster and service on the cluster + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/eks/deployment + fi + + terraform init + if terraform apply --auto-approve \ + -var="test_dir=${{ matrix.arrays.test_dir }}"\ + -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ + -var="cwagent_image_tag=${{ github.sha }}" \ + -var="k8s_version=${{ matrix.arrays.k8s_version }}"; then + terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + + - name: Terraform destroy + if: ${{ cancelled() || failure() }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/eks/deployment + fi + terraform destroy --auto-approve + + PerformanceTrackingTest: + name: "PerformanceTrackingTest" + needs: [BuildAndUpload, GenerateTestMatrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_performance_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Cache if success + id: performance-tracking + uses: actions/cache@v3 + with: + path: go.mod + key: performance-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} + + - name: Verify Terraform version + if: steps.performance-tracking.outputs.cache-hit != 'true' + run: terraform --version + + - name: Terraform apply + if: steps.performance-tracking.outputs.cache-hit != 'true' + uses: nick-fields/retry@v2 + with: + max_attempts: 1 + timeout_minutes: 60 + retry_wait_seconds: 5 + command: | + cd terraform/performance + terraform init + if terraform apply --auto-approve \ + -var="ssh_key_value=${PRIVATE_KEY}" \ + -var="cwa_github_sha=${GITHUB_SHA}" \ + -var="ami=${{ matrix.arrays.ami }}" \ + -var="arc=${{ matrix.arrays.arc }}" \ + -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ + -var="ssh_key_name=${KEY_NAME}" \ + -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ + -var="family=${{ matrix.arrays.family}}"\ + -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + + - name: Terraform destroy + if: ${{ cancelled() || failure() }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: cd terraform/performance && terraform destroy --auto-approve + + EC2WinPerformanceTest: + name: "EC2WinPerformanceTest" + needs: [ BuildAndUpload, BuildAndUploadPackages, GenerateTestMatrix ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_performance_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Cache if success + id: performance-tracking + uses: actions/cache@v3 + with: + path: go.mod + key: performance-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} + + - name: Verify Terraform version + if: steps.performance-tracking.outputs.cache-hit != 'true' + run: terraform --version + + - name: Terraform apply + if: steps.performance-tracking.outputs.cache-hit != 'true' + uses: nick-fields/retry@v2 + with: + max_attempts: 1 + timeout_minutes: 60 + retry_wait_seconds: 5 + command: | + cd terraform/performance + terraform init + if terraform apply --auto-approve \ + -var="ssh_key_value=${PRIVATE_KEY}" \ + -var="cwa_github_sha=${GITHUB_SHA}" \ + -var="ami=${{ matrix.arrays.ami }}" \ + -var="arc=${{ matrix.arrays.arc }}" \ + -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ + -var="ssh_key_name=${KEY_NAME}" \ + -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ + -var="family=${{ matrix.arrays.family}}"\ + -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + + - name: Terraform destroy + if: ${{ cancelled() || failure() }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: cd terraform/performance && terraform destroy --auto-approve + + StressTrackingTest: + name: "StressTrackingTest" + needs: [BuildAndUpload, GenerateTestMatrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_stress_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Cache if success + id: stress-tracking + uses: actions/cache@v3 + with: + path: go.mod + key: stress-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} + + - name: Verify Terraform version + if: steps.stress-tracking.outputs.cache-hit != 'true' + run: terraform --version + + - name: Echo Test Info + run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} + + - name: Terraform apply + if: steps.stress-tracking.outputs.cache-hit != 'true' + uses: nick-fields/retry@v2 + with: + max_attempts: 1 + timeout_minutes: 60 + retry_wait_seconds: 5 + command: | + cd terraform/stress + terraform init + if terraform apply --auto-approve \ + -var="ssh_key_value=${PRIVATE_KEY}" \ + -var="cwa_github_sha=${GITHUB_SHA}" \ + -var="ami=${{ matrix.arrays.ami }}" \ + -var="arc=${{ matrix.arrays.arc }}" \ + -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ + -var="ssh_key_name=${KEY_NAME}" \ + -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ + -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + + - name: Terraform destroy + if: ${{ cancelled() || failure() }} + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: cd terraform/stress && terraform destroy --auto-approve + + EC2WinStressTrackingTest: + name: "EC2WinStressTrackingTest" + needs: [BuildAndUpload, BuildAndUploadPackages, GenerateTestMatrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_stress_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Cache if success + id: ec2-win-stress-tracking-test + uses: actions/cache@v3 + with: + path: go.mod + key: ec2-win-stress-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} + + - name: Verify Terraform version + if: steps.ec2-win-stress-tracking-test.outputs.cache-hit != 'true' + run: terraform --version + + - name: Echo Test Info + run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} + + - name: Terraform apply + if: steps.ec2-win-stress-tracking-test.outputs.cache-hit != 'true' + uses: nick-fields/retry@v2 + with: + max_attempts: 1 + timeout_minutes: 60 + retry_wait_seconds: 5 + command: | + cd terraform/stress + terraform init + if terraform apply --auto-approve \ + -var="ssh_key_value=${PRIVATE_KEY}" \ + -var="cwa_github_sha=${GITHUB_SHA}" \ + -var="ami=${{ matrix.arrays.ami }}" \ + -var="arc=${{ matrix.arrays.arc }}" \ + -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ -var="ssh_key_name=${KEY_NAME}" \ -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ -var="family=${{ matrix.arrays.family}}"\ @@ -1301,106 +1301,106 @@ jobs: timeout_minutes: 8 retry_wait_seconds: 5 command: cd terraform/stress && terraform destroy --auto-approve -# -# ApplicationSignalsEndToEndTest: -# name: "Application Signals E2E Test" -# needs: [ BuildAndUpload, BuildDocker ] -# uses: ./.github/workflows/application-signals-e2e-test.yml -# secrets: inherit -# permissions: -# id-token: write -# contents: read -# -# GPUEndToEndTest: -# name: "GPU E2E Test" -# needs: [ BuildAndUpload, StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_addon_matrix) }} -# permissions: -# id-token: write -# contents: read -# steps: -# - uses: actions/checkout@v3 -# with: -# repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} -# ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} -# aws-region: us-west-2 -# role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} -# -# -# - name: Verify Terraform version -# run: terraform --version -# -# -# - name: Terraform apply and setup -# run: | -# if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then -# cd "${{ matrix.arrays.terraform_dir }}" -# else -# cd terraform/eks/addon/gpu -# fi -# -# terraform init -# if terraform apply --auto-approve \ -# -var="beta=true" \ -# -var="addon_name=amazon-cloudwatch-observability" \ -# -var="addon_version=v1.6.0-eksbuild.1" \ -# -var="k8s_version=1.29" ; then -# echo "Terraform apply successful." -# -# # Capture the output -# echo "Getting EKS cluster name" -# EKS_CLUSTER_NAME=$(terraform output -raw eks_cluster_name) -# echo "Cluster name is ${EKS_CLUSTER_NAME}" -# kubectl apply -f ./gpuBurner.yaml -# kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.15.0/deployments/static/nvidia-device-plugin.yml -# kubectl patch amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": ${{ secrets.AWS_ECR_PRIVATE_REGISTRY }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }}}]' -# else -# terraform destroy -var="beta=${{ github.event.inputs.run_in_beta }}" -auto-approve && exit 1 -# fi -# -# - name: Run Go tests with retry -# uses: nick-fields/retry@v2 -# with: -# max_attempts: 10 -# timeout_minutes: 60 -# retry_wait_seconds: 60 -# command: | -# if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then -# cd "${{ matrix.arrays.terraform_dir }}" -# else -# cd terraform/eks/addon/gpu -# fi -# echo "Getting EKS cluster name" -# EKS_CLUSTER_NAME=$(terraform output -raw eks_cluster_name) -# echo "Cluster name is ${EKS_CLUSTER_NAME}" -# -# if go test ${{ matrix.arrays.test_dir }} -eksClusterName ${EKS_CLUSTER_NAME} -computeType=EKS -v -eksDeploymentStrategy=DAEMON -eksGpuType=nvidia -useE2EMetrics; then -# echo "Tests passed" -# else -# echo "Tests failed" -# exit 1 -# fi -# -# - name: Terraform destroy -# if: always() -# uses: nick-fields/retry@v2 -# with: -# max_attempts: 3 -# timeout_minutes: 8 -# retry_wait_seconds: 5 -# command: | -# if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then -# cd "${{ matrix.arrays.terraform_dir }}" -# else -# cd terraform/eks/addon/gpu -# fi -# terraform destroy --auto-approve \ No newline at end of file + + ApplicationSignalsEndToEndTest: + name: "Application Signals E2E Test" + needs: [ BuildAndUpload, BuildDocker ] + uses: ./.github/workflows/application-signals-e2e-test.yml + secrets: inherit + permissions: + id-token: write + contents: read + + GPUEndToEndTest: + name: "GPU E2E Test" + needs: [ BuildAndUpload, StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_addon_matrix) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} + ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + + - name: Verify Terraform version + run: terraform --version + + + - name: Terraform apply and setup + run: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/eks/addon/gpu + fi + + terraform init + if terraform apply --auto-approve \ + -var="beta=true" \ + -var="addon_name=amazon-cloudwatch-observability" \ + -var="addon_version=v1.6.0-eksbuild.1" \ + -var="k8s_version=1.29" ; then + echo "Terraform apply successful." + + # Capture the output + echo "Getting EKS cluster name" + EKS_CLUSTER_NAME=$(terraform output -raw eks_cluster_name) + echo "Cluster name is ${EKS_CLUSTER_NAME}" + kubectl apply -f ./gpuBurner.yaml + kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.15.0/deployments/static/nvidia-device-plugin.yml + kubectl patch amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": ${{ secrets.AWS_ECR_PRIVATE_REGISTRY }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }}}]' + else + terraform destroy -var="beta=${{ github.event.inputs.run_in_beta }}" -auto-approve && exit 1 + fi + + - name: Run Go tests with retry + uses: nick-fields/retry@v2 + with: + max_attempts: 10 + timeout_minutes: 60 + retry_wait_seconds: 60 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/eks/addon/gpu + fi + echo "Getting EKS cluster name" + EKS_CLUSTER_NAME=$(terraform output -raw eks_cluster_name) + echo "Cluster name is ${EKS_CLUSTER_NAME}" + + if go test ${{ matrix.arrays.test_dir }} -eksClusterName ${EKS_CLUSTER_NAME} -computeType=EKS -v -eksDeploymentStrategy=DAEMON -eksGpuType=nvidia -useE2EMetrics; then + echo "Tests passed" + else + echo "Tests failed" + exit 1 + fi + + - name: Terraform destroy + if: always() + uses: nick-fields/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: | + if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then + cd "${{ matrix.arrays.terraform_dir }}" + else + cd terraform/eks/addon/gpu + fi + terraform destroy --auto-approve \ No newline at end of file