diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 222e465bd4..35aa64a8ae 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -184,157 +184,6 @@ jobs: github_sha: ${{github.sha}} s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - StartLocalStackITAR: - name: 'StartLocalStackITAR' - needs: [OutputEnvVariables] - uses: ./.github/workflows/start-localstack.yml - secrets: inherit - permissions: - id-token: write - contents: read - with: - region: us-gov-east-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - github_sha: ${{github.sha}} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} - - StartLocalStackCN: - name: 'StartLocalStackCN' - needs: [ OutputEnvVariables ] - uses: ./.github/workflows/start-localstack.yml - secrets: inherit - permissions: - id-token: write - contents: read - with: - region: cn-north-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - github_sha: ${{github.sha}} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - - - - EC2NvidiaGPUIntegrationTest: - needs: [ StartLocalStack, GenerateTestMatrix ] - name: 'EC2NVIDIAGPUIntegrationTest' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_gpu_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: ec2-linux-integration-test - uses: actions/cache@v3 - with: - path: go.mod - key: ec2-nvidia-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} - - - name: Verify Terraform version - run: terraform --version - - # nick-fields/retry@v2 starts at base dir - - name: Terraform apply - if: ${{ matrix.arrays.family == 'linux' && steps.ec2-nvidia-integration-test.outputs.cache-hit != 'true' }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 30 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/linux - fi - - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - -var="test_name=${{ matrix.arrays.os }}" \ - -var="cwa_github_sha=${GITHUB_SHA}" -var="install_agent=${{ matrix.arrays.installAgentCommand }}" \ - -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" \ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="user=${{ matrix.arrays.username }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="ca_cert_path=${{ matrix.arrays.caCertPath }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="binary_name=${{ matrix.arrays.binaryName }}" \ - -var="local_stack_host_name=${{ needs.StartLocalStack.outputs.local_stack_host_name }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - name: Terraform apply - if: ${{ matrix.arrays.family == 'window' && steps.ec2-nvidia-integration-test.outputs.cache-hit != 'true' }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 30 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/win - fi - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ - -var="github_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - -var="cwa_github_sha=${GITHUB_SHA}" -var="ami=${{ matrix.arrays.ami }}" \ - -var="test_dir=${{ matrix.arrays.test_dir }}" \ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - #This is here just in case workflow cancel - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - elif if "${{ matrix.arrays.os }}" == window; then - cd terraform/ec2/win - else - cd terraform/ec2/linux - fi - terraform destroy --auto-approve - OutputEnvVariables: needs: [CheckBuildTestArtifacts] name: 'OutputEnvVariables' @@ -367,159 +216,40 @@ jobs: echo "CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}" echo "CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}" - EC2LinuxIntegrationTest: - needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] - name: 'EC2Linux' - uses: ./.github/workflows/ec2-integration-test.yml + LinuxOnPremIntegrationTest: + needs: [StartLocalStack, GenerateTestMatrix, OutputEnvVariables] + name: 'OnpremLinux' + uses: ./.github/workflows/ec2-integration-test.yml with: github_sha: ${{github.sha}} - test_dir: terraform/ec2/linux - job_id: ec2-linux-integration-test - test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_matrix}} + test_dir: terraform/ec2/linux_onprem + job_id: linux-onprem-integration-test + test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_onprem_matrix}} test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} region: us-west-2 - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - secrets: inherit - - EC2LinuxIntegrationTestITAR: - needs: [ StartLocalStackITAR, GenerateTestMatrix, OutputEnvVariables ] - name: 'EC2LinuxITAR' - uses: ./.github/workflows/ec2-integration-test.yml - with: - github_sha: ${{github.sha}} - test_dir: terraform/ec2/linux - job_id: ec2-linux-integration-test - test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_itar_matrix}} - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - localstack_host: ${{needs.StartLocalStackITAR.outputs.local_stack_host_name}} - region: us-gov-east-1 - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} secrets: inherit - EC2LinuxIntegrationTestCN: - needs: [ StartLocalStackCN, GenerateTestMatrix, OutputEnvVariables ] - name: 'EC2LinuxCN' - uses: ./.github/workflows/ec2-integration-test.yml + EC2LinuxIntegrationTest: + needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] + name: 'EC2Linux' + uses: ./.github/workflows/ec2-integration-test.yml with: github_sha: ${{github.sha}} test_dir: terraform/ec2/linux job_id: ec2-linux-integration-test - test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_china_matrix}} - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - localstack_host: ${{needs.StartLocalStackCN.outputs.local_stack_host_name}} - region: cn-north-1 - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - secrets: inherit - - - LinuxOnPremIntegrationTest: - needs: [StartLocalStack, GenerateTestMatrix, OutputEnvVariables] - name: 'OnpremLinux' - uses: ./.github/workflows/ec2-integration-test.yml - with: - github_sha: ${{github.sha}} - test_dir: terraform/ec2/linux_onprem - job_id: linux-onprem-integration-test - test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_onprem_matrix}} + test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_matrix}} test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} region: us-west-2 + terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} + s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} secrets: inherit - EC2WinIntegrationTest: - needs: [GenerateTestMatrix] - name: 'EC2WinIntegrationTest' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: ec2-win-integration-test - uses: actions/cache@v3 - with: - path: go.mod - key: ec2-win-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} use ssm ${{ matrix.arrays.useSSM }} test ${{ matrix.arrays.test_dir }} - - - name: Verify Terraform version - run: terraform --version - - # nick-fields/retry@v2 starts at base dir - - name: Terraform apply - if: steps.ec2-win-integration-test.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 30 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/win - fi - terraform init - if terraform apply --auto-approve \ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ - -var="test_name=${{ matrix.arrays.os }}" \ - -var="cwa_github_sha=${GITHUB_SHA}" \ - -var="test_dir=${{ matrix.arrays.test_dir }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="use_ssm=${{ matrix.arrays.useSSM }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - #This is here just in case workflow cancel - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/win - fi - terraform destroy --auto-approve - EC2DarwinIntegrationTest: needs: [GenerateTestMatrix] name: 'EC2DarwinIntegrationTest' @@ -616,732 +346,4 @@ jobs: test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} github_sha: ${{github.sha}} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - - StopLocalStackITAR: - name: 'StopLocalStackITAR' - if: ${{ always() && needs.StartLocalStackITAR.result == 'success' }} - needs: [ StartLocalStackITAR, EC2LinuxIntegrationTestITAR, OutputEnvVariables ] - uses: ./.github/workflows/stop-localstack.yml - secrets: inherit - permissions: - id-token: write - contents: read - with: - region: us-gov-east-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - github_sha: ${{github.sha}} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} - - StopLocalStackCN: - name: 'StopLocalStackCN' - if: ${{ always() && needs.StartLocalStackCN.result == 'success' }} - needs: [ StartLocalStackCN, EC2LinuxIntegrationTestCN ] - uses: ./.github/workflows/stop-localstack.yml - secrets: inherit - permissions: - id-token: write - contents: read - with: - region: cn-north-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - github_sha: ${{github.sha}} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - - ECSEC2IntegrationTest: - name: 'ECSEC2IntegrationTest' - runs-on: ubuntu-latest - needs: [ GenerateTestMatrix ] - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_ec2_launch_daemon_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: ecs-ec2-integration-test - uses: actions/cache@v3 - with: - path: go.mod - key: ecs-ec2-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} - - - name: Login ECR - id: login-ecr - if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' - uses: aws-actions/amazon-ecr-login@v2 - - - name: Verify Terraform version - if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' - run: terraform --version - - - name: Terraform apply - if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 15 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ecs_ec2/daemon - fi - - terraform init - if terraform apply --auto-approve\ - -var="test_dir=${{ matrix.arrays.test_dir }}"\ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ - -var="cwagent_image_tag=${{ github.sha }}"\ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="metadataEnabled=${{ matrix.arrays.metadataEnabled }}" \ - -var="ami=${{ matrix.arrays.ami }}" ; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ecs_ec2/daemon - fi - terraform destroy --auto-approve - - ECSFargateIntegrationTest: - name: 'ECSFargateIntegrationTest' - runs-on: ubuntu-latest - needs: [GenerateTestMatrix] - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_fargate_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: ecs-fargate-integration-test - uses: actions/cache@v3 - with: - path: go.mod - key: ecs-fargate-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} - - - name: Login ECR - id: login-ecr - if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' - uses: aws-actions/amazon-ecr-login@v2 - - - name: Verify Terraform version - if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' - run: terraform --version - - - name: Terraform apply - if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 15 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ecs_fargate/linux - fi - - terraform init - if terraform apply --auto-approve\ - -var="test_dir=${{ matrix.arrays.test_dir }}"\ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ - -var="cwagent_image_tag=${{ github.sha }}"; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ecs_fargate/linux - fi - terraform destroy --auto-approve - - EKSIntegrationTest: - name: 'EKSIntegrationTest' - runs-on: ubuntu-latest - needs: [ GenerateTestMatrix ] - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_daemon_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: eks-ec2-integration-test - uses: actions/cache@v3 - with: - path: go.mod - key: ${{ matrix.arrays.terraform_dir }}-${{ matrix.arrays.k8s_version }}-${{ matrix.arrays.instanceType }}-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} - - - name: Login ECR - id: login-ecr - if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - uses: aws-actions/amazon-ecr-login@v2 - - - name: Verify Terraform version - if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - run: terraform --version - - - name: Terraform apply - if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 2 - timeout_minutes: 90 # EKS takes about 20 minutes to spin up a cluster and service on the cluster - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/daemon - fi - - terraform init - if terraform apply --auto-approve \ - -var="test_dir=${{ matrix.arrays.test_dir }}"\ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ - -var="cwagent_image_tag=${{ github.sha }}" \ - -var="ami_type=${{ matrix.arrays.ami }}" \ - -var="instance_type=${{ matrix.arrays.instanceType }}" \ - -var="k8s_version=${{ matrix.arrays.k8s_version }}"; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/daemon - fi - terraform destroy --auto-approve - - EKSPrometheusIntegrationTest: - name: 'EKSPrometheusIntegrationTest' - runs-on: ubuntu-latest - needs: [ GenerateTestMatrix ] - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_deployment_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: eks-ec2-integration-test - uses: actions/cache@v3 - with: - path: go.mod - key: eks-ec2-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} - - - name: Login ECR - id: login-ecr - if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - uses: aws-actions/amazon-ecr-login@v2 - - - name: Verify Terraform version - if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - run: terraform --version - - - name: Terraform apply - if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 60 # EKS takes about 20 minutes to spin up a cluster and service on the cluster - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/deployment - fi - - terraform init - if terraform apply --auto-approve \ - -var="test_dir=${{ matrix.arrays.test_dir }}"\ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ - -var="cwagent_image_tag=${{ github.sha }}" \ - -var="k8s_version=${{ matrix.arrays.k8s_version }}"; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/deployment - fi - terraform destroy --auto-approve - - PerformanceTrackingTest: - name: "PerformanceTrackingTest" - needs: [GenerateTestMatrix] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_performance_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: performance-tracking - uses: actions/cache@v3 - with: - path: go.mod - key: performance-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - - - name: Verify Terraform version - if: steps.performance-tracking.outputs.cache-hit != 'true' - run: terraform --version - - - name: Terraform apply - if: steps.performance-tracking.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 1 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - cd terraform/performance - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${GITHUB_SHA}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="family=${{ matrix.arrays.family}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: cd terraform/performance && terraform destroy --auto-approve - - EC2WinPerformanceTest: - name: "EC2WinPerformanceTest" - needs: [ GenerateTestMatrix ] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_performance_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: performance-tracking - uses: actions/cache@v3 - with: - path: go.mod - key: performance-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - - - name: Verify Terraform version - if: steps.performance-tracking.outputs.cache-hit != 'true' - run: terraform --version - - - name: Terraform apply - if: steps.performance-tracking.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 1 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - cd terraform/performance - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${GITHUB_SHA}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="family=${{ matrix.arrays.family}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: cd terraform/performance && terraform destroy --auto-approve - - StressTrackingTest: - name: "StressTrackingTest" - needs: [GenerateTestMatrix] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_stress_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: stress-tracking - uses: actions/cache@v3 - with: - path: go.mod - key: stress-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - - - name: Verify Terraform version - if: steps.stress-tracking.outputs.cache-hit != 'true' - run: terraform --version - - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} - - - name: Terraform apply - if: steps.stress-tracking.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 1 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - cd terraform/stress - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${GITHUB_SHA}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: cd terraform/stress && terraform destroy --auto-approve - - EC2WinStressTrackingTest: - name: "EC2WinStressTrackingTest" - needs: [GenerateTestMatrix] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_stress_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Cache if success - id: ec2-win-stress-tracking-test - uses: actions/cache@v3 - with: - path: go.mod - key: ec2-win-stress-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} - - - name: Verify Terraform version - if: steps.ec2-win-stress-tracking-test.outputs.cache-hit != 'true' - run: terraform --version - - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} - - - name: Terraform apply - if: steps.ec2-win-stress-tracking-test.outputs.cache-hit != 'true' - uses: nick-fields/retry@v2 - with: - max_attempts: 1 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - cd terraform/stress - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${GITHUB_SHA}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="family=${{ matrix.arrays.family}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: cd terraform/stress && terraform destroy --auto-approve - - GPUEndToEndTest: - name: "GPU E2E Test" - needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_addon_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} - ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - - name: Verify Terraform version - run: terraform --version - - - - name: Terraform apply and setup - run: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/addon/gpu - fi - - terraform init - if terraform apply --auto-approve \ - -var="beta=true" \ - -var="addon_name=amazon-cloudwatch-observability" \ - -var="addon_version=v1.6.0-eksbuild.1" \ - -var="k8s_version=1.29" ; then - echo "Terraform apply successful." - - # Capture the output - echo "Getting EKS cluster name" - EKS_CLUSTER_NAME=$(terraform output -raw eks_cluster_name) - echo "Cluster name is ${EKS_CLUSTER_NAME}" - kubectl apply -f ./gpuBurner.yaml - kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.15.0/deployments/static/nvidia-device-plugin.yml - kubectl patch amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": ${{ secrets.AWS_ECR_PRIVATE_REGISTRY }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }}}]' - else - terraform destroy -var="beta=${{ github.event.inputs.run_in_beta }}" -auto-approve && exit 1 - fi - - - name: Run Go tests with retry - uses: nick-fields/retry@v2 - with: - max_attempts: 10 - timeout_minutes: 60 - retry_wait_seconds: 60 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/addon/gpu - fi - echo "Getting EKS cluster name" - EKS_CLUSTER_NAME=$(terraform output -raw eks_cluster_name) - echo "Cluster name is ${EKS_CLUSTER_NAME}" - - if go test ${{ matrix.arrays.test_dir }} -eksClusterName ${EKS_CLUSTER_NAME} -computeType=EKS -v -eksDeploymentStrategy=DAEMON -eksGpuType=nvidia -useE2EMetrics; then - echo "Tests passed" - else - echo "Tests failed" - exit 1 - fi - - - name: Terraform destroy - if: always() - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/addon/gpu - fi - terraform destroy --auto-approve + s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} \ No newline at end of file