dotnet-ec2-windows-test.yml

## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
## SPDX-License-Identifier: Apache-2.0

# This is a reusable workflow for running the DotNet E2E Canary test for Application Signals.
# It is meant to be called from another workflow.
# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
name: DotNet EC2 Windows Use Case
on:
  workflow_call:
    inputs:
      aws-region:
        required: true
        type: string
      staging_distro_name:
        required: false
        default: 'aws-opentelemetry-distro'
        type: string
      caller-workflow-name:
        required: true
        type: string
    outputs:
      job-started:
        value: ${{ jobs.dotnet-ec2-windows.outputs.job-started }}
      validation-result:
        value: ${{ jobs.dotnet-ec2-windows.outputs.validation-result }}

permissions:
  id-token: write
  contents: read

env:
  E2E_TEST_AWS_REGION: ${{ inputs.aws-region }}
  E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
  E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
  SAMPLE_APP_ZIP: "aws s3 cp s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/dotnet-sample-app.zip ./dotnet-sample-app.zip"
  METRIC_NAMESPACE: ApplicationSignals
  LOG_GROUP_NAME: /aws/application-signals/data
  ADOT_DISTRO_NAME: ${{ inputs.staging_distro_name }}
  TEST_RESOURCES_FOLDER: ${GITHUB_WORKSPACE}

jobs:
  dotnet-ec2-windows:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    outputs:
      job-started: ${{ steps.job-started.outputs.job-started }}
      validation-result: ${{ steps.validation-result.outputs.validation-result }}
    steps:
      - name: Check if the job started
        id: job-started
        run: echo "job-started=true" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v4
        with:
          repository: 'aws-observability/aws-application-signals-test-framework'
          ref: ${{ inputs.caller-workflow-name == 'main-build' && 'main' || github.ref }}
          fetch-depth: 0

      - name: Initiate Gradlew Daemon
        id: initiate-gradlew
        uses: ./.github/workflows/actions/execute_and_retry
        continue-on-error: true
        with:
          command: "./gradlew :validator:build"
          cleanup: "./gradlew clean"
          max_retry: 3
          sleep_time: 60

      - name: Generate testing id
        run: echo TESTING_ID="${{ github.job }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" >> $GITHUB_ENV

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ env.E2E_TEST_ACCOUNT_ID }}:role/${{ env.E2E_TEST_ROLE_NAME }}
          aws-region: us-east-1

      - name: Retrieve account
        uses: aws-actions/aws-secretsmanager-get-secrets@v1
        with:
          secret-ids:
            ACCOUNT_ID, region-account/${{ env.E2E_TEST_AWS_REGION }}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ env.ACCOUNT_ID }}:role/${{ env.E2E_TEST_ROLE_NAME }}
          aws-region: ${{ env.E2E_TEST_AWS_REGION }}

      - name: Set Get ADOT Distro command environment variable
        run: |
          if [ "${{ github.event.repository.name }}" = "aws-otel-dotnet-instrumentation" ]; then
            # Get staging distro file from staging bucket
            echo GET_ADOT_DISTRO_COMMAND="aws s3 cp s3://adot-autoinstrumentation-dotnet-staging/${{ env.ADOT_DISTRO_NAME }} ./${{ env.ADOT_DISTRO_NAME }}; Expand-Archive -Path ${{ env.ADOT_DISTRO_NAME }} -DestinationPath ./dotnet-distro" >> $GITHUB_ENV
          else
            # After Release will switch to latest tag instead of hard code version for canary purpose
            echo GET_ADOT_DISTRO_COMMAND="curl.exe -L -o ./aws-distro-opentelemetry-dotnet-instrumentation-windows.zip https://github.com/aws-observability/aws-otel-dotnet-instrumentation/releases/latest/download/aws-distro-opentelemetry-dotnet-instrumentation-windows.zip --retry 5 --retry-all-errors --retry-delay 5; Expand-Archive -Path ./aws-distro-opentelemetry-dotnet-instrumentation-windows.zip -DestinationPath ./dotnet-distro -Force" >> $GITHUB_ENV
          fi

      - name: Set Get CW Agent command environment variable
        run: |
          if [ "${{ github.event.repository.name }}" = "amazon-cloudwatch-agent" ]; then
            # Get cloudwatch agent staging file if triggered by cw-a repo
            echo GET_CW_AGENT_MSI_COMMAND= "aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/integration-test/binary/${{ github.sha }}/amazon-cloudwatch-agent.msi ./cw-agent.msi" >> $GITHUB_ENV
          else
            echo GET_CW_AGENT_MSI_COMMAND= "curl.exe -L -o ./amazon-cloudwatch-agent.msi https://amazoncloudwatch-agent.s3.amazonaws.com/windows/amd64/latest/amazon-cloudwatch-agent.msi --retry 5 --retry-all-errors --retry-delay 5" >> $GITHUB_ENV
          fi

      - name: Set up terraform
        uses: ./.github/workflows/actions/execute_and_retry
        with:
          command: "wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg"
          post-command: 'echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
              && sudo apt update && sudo apt install terraform'
          sleep_time: 60

      - name: Initiate Terraform
        uses: ./.github/workflows/actions/execute_and_retry
        with:
          command: "cd ${{ env.TEST_RESOURCES_FOLDER }}/terraform/dotnet/ec2/windows && terraform init && terraform validate"
          cleanup: "rm -rf .terraform && rm -rf .terraform.lock.hcl"
          max_retry: 6
          sleep_time: 60

      - name: Deploy sample app via terraform and wait for endpoint to come online
        working-directory: terraform/dotnet/ec2/windows
        run: |
          # Attempt to deploy the sample app on an EC2 instance and wait for its endpoint to come online. 
          # There may be occasional failures due to transitivity issues, so try up to 2 times. 
          # deployment_failed of 0 indicates that both the terraform deployment and the endpoint are running, while 1 indicates
          # that it failed at some point
          retry_counter=0
          max_retry=2
          while [ $retry_counter -lt $max_retry ]; do
            echo "Attempt $retry_counter"
            deployment_failed=0
            terraform apply -auto-approve \
              -var="aws_region=${{ env.E2E_TEST_AWS_REGION }}" \
              -var="test_id=${{ env.TESTING_ID }}" \
              -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \
              -var="get_cw_agent_msi_command=${{ env.GET_CW_AGENT_MSI_COMMAND }}" \
              -var="get_adot_distro_command=${{ env.GET_ADOT_DISTRO_COMMAND }}" \
            || deployment_failed=$?
          
            if [ $deployment_failed -eq 1 ]; then
              echo "Terraform deployment was unsuccessful. Will attempt to retry deployment."
            fi
          
            # If the success is 1 then either the terraform deployment or the endpoint connection failed, so first destroy the
            # resources created from terraform and try again.
            if [ $deployment_failed -eq 1 ]; then
              echo "Destroying terraform"
              terraform destroy -auto-approve \
                -var="test_id=${{ env.TESTING_ID }}" 
          
              retry_counter=$(($retry_counter+1))
            else
              # If deployment succeeded, then exit the loop
              break
            fi
          
            if [ $retry_counter -eq $max_retry ]; then
              echo "Max retry reached, failed to deploy terraform and connect to the endpoint. Exiting code"
              exit 1
            fi
          done

      - name: Get the ec2 instance ami id
        working-directory: terraform/dotnet/ec2/windows
        run: |
          echo "EC2_INSTANCE_AMI=$(terraform output ec2_instance_ami)" >> $GITHUB_ENV

      - name: Get SSM outputs
        working-directory: terraform/dotnet/ec2/windows
        run: |
          echo "FRONTEND_DOCUMENT_NAME=$(terraform output frontend_document_name)" >> $GITHUB_ENV
          echo "REMOTE_DOCUMENT_NAME=$(terraform output remote_document_name)" >> $GITHUB_ENV

      - name: Get the sample app endpoint
        working-directory: terraform/dotnet/ec2/windows
        run: |
          echo "MAIN_SERVICE_ENDPOINT=localhost:8080" >> $GITHUB_ENV
          echo "REMOTE_SERVICE_IP=$(terraform output sample_app_remote_service_private_ip)" >> $GITHUB_ENV
          echo "MAIN_SERVICE_INSTANCE_ID=$(terraform output main_service_instance_id)" >> $GITHUB_ENV
          echo "REMOTE_SERVICE_INSTANCE_ID=$(terraform output remote_service_instance_id)" >> $GITHUB_ENV

# A standard Windows Deployment Script are Expect to run ~5min, plus waiting for SSM to be setup in EC2 instance ~2min
# 15 Min loose upper timeout bound is designed to allow for a rate situation: in a very small chance,
# It's possible for SSM document to take up to 8min to run and still produce expect setup outcome
      - name: Monitor EC2 instances readiness and SSM execute status in Systems Manager
        timeout-minutes: 18
        run: |
          # Allow up to 5 min for EC2 instance connect to SSM
          max_attempts=30
          attempt=0
          interval=10

          check_instance_ready() {
            local instance_id=$1
            while [[ $attempt -lt $max_attempts ]]; do
              state=$(aws ssm describe-instance-information --filters Key=InstanceIds,Values=$instance_id --query "InstanceInformationList[0].PingStatus" --output text)
              echo "Attempt $((attempt + 1))/$max_attempts: Current SSM state of instance $instance_id: $state"

              if [[ "$state" == "Online" ]]; then
                echo "Instance $instance_id is 'Online' in Systems Manager and ready."
                break
              else
                echo "Waiting for instance $instance_id to be 'Online' in Systems Manager..."
                sleep $interval
                attempt=$((attempt + 1))
              fi
            done

            if [[ $attempt -ge $max_attempts ]]; then
              echo "Timeout reached: Instance $instance_id did not become 'Online' in Systems Manager within expected time."
              exit 1
            fi
          }

          check_instance_ready ${{ env.MAIN_SERVICE_INSTANCE_ID }}
          check_instance_ready ${{ env.REMOTE_SERVICE_INSTANCE_ID }}
          
          main_command_id=$(aws ssm send-command \
              --instance-ids "${{ env.MAIN_SERVICE_INSTANCE_ID }}" \
              --document-name "${{ env.FRONTEND_DOCUMENT_NAME }}" \
              --query "Command.CommandId" \
              --output text)
          
          remote_command_id=$(aws ssm send-command \
              --instance-ids "${{ env.REMOTE_SERVICE_INSTANCE_ID }}" \
              --document-name "${{ env.REMOTE_DOCUMENT_NAME }}" \
              --query "Command.CommandId" \
              --output text)
          
          watch_command() {
            local command_id=$1
            local max_attempts=30  # 15 minutes timeout with 30 seconds interval
            local attempt=0
            local interval=30

            while [[ $attempt -lt $max_attempts ]]; do
              status=$(aws ssm list-command-invocations \
                --command-id "$command_id" \
                --details \
                --query "CommandInvocations[0].Status" \
                --output text)

              echo "Attempt $((attempt + 1))/$max_attempts: Current status of command $command_id: $status"

              if [[ "$status" == "Success" || "$status" == "Failed" ]]; then
                echo "SSM Command $command_id completed with status: $status"
                if [[ "$status" == "Failed" ]]; then
                  exit 1
                fi
                break
              else
                echo "Waiting for SSM Command $command_id to complete..."
                sleep $interval
                attempt=$((attempt + 1))
              fi
            done

            if [[ $attempt -ge $max_attempts ]]; then
              echo "Timeout reached: SSM Command $command_id did not complete within 10 minutes."
              exit 1
            fi
          }
          
          watch_command $main_command_id
          watch_command $remote_command_id

      - name: Initiate Gradlew Daemon
        if: steps.initiate-gradlew == 'failure'
        uses: ./.github/workflows/actions/execute_and_retry
        continue-on-error: true
        with:
          command: "./gradlew"
          cleanup: "./gradlew clean"
          max_retry: 3
          sleep_time: 60

      # Validation for pulse telemetry data
      - name: Validate generated EMF logs
        id: log-validation
        run: ./gradlew validator:run --args='-c dotnet/ec2/windows/log-validation.yml
          --testing-id ${{ env.TESTING_ID }}
          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8081
          --region ${{ env.E2E_TEST_AWS_REGION }}
          --metric-namespace ${{ env.METRIC_NAMESPACE }}
          --log-group ${{ env.LOG_GROUP_NAME }}
          --service-name dotnet-sample-application-${{ env.TESTING_ID }}
          --remote-service-name dotnet-sample-remote-application-${{ env.TESTING_ID }}
          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
          --rollup'

      - name: Validate generated metrics
        id: metric-validation
        if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
        run: ./gradlew validator:run --args='-c dotnet/ec2/windows/metric-validation.yml
          --testing-id ${{ env.TESTING_ID }}
          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8081
          --region ${{ env.E2E_TEST_AWS_REGION }}
          --metric-namespace ${{ env.METRIC_NAMESPACE }}
          --log-group ${{ env.LOG_GROUP_NAME }}
          --service-name dotnet-sample-application-${{ env.TESTING_ID }}
          --remote-service-name dotnet-sample-remote-application-${{ env.TESTING_ID }}
          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
          --rollup'

      - name: Validate generated traces
        id: trace-validation
        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
        run: ./gradlew validator:run --args='-c dotnet/ec2/windows/trace-validation.yml
          --testing-id ${{ env.TESTING_ID }}
          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8081
          --region ${{ env.E2E_TEST_AWS_REGION }}
          --account-id ${{ env.ACCOUNT_ID }}
          --metric-namespace ${{ env.METRIC_NAMESPACE }}
          --log-group ${{ env.LOG_GROUP_NAME }}
          --service-name dotnet-sample-application-${{ env.TESTING_ID }}
          --remote-service-name dotnet-sample-remote-application-${{ env.TESTING_ID }}
          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
          --rollup'

      - name: Refresh AWS Credentials
        if: ${{ github.event.repository.name == 'aws-application-signals-test-framework' }}
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ env.ACCOUNT_ID }}:role/${{ env.E2E_TEST_ROLE_NAME }}
          aws-region: ${{ env.E2E_TEST_AWS_REGION }}

      - name: Save test results
        if: always()
        id: validation-result
        run: |
          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
            echo "validation-result=success" >> $GITHUB_OUTPUT
          else
            echo "validation-result=failure" >> $GITHUB_OUTPUT
          fi

      # Clean up Procedures
      - name: Terraform destroy
        if: always()
        continue-on-error: true
        working-directory: terraform/dotnet/ec2/windows
        run: |
          terraform destroy -auto-approve \
            -var="test_id=${{ env.TESTING_ID }}"