Add Python Version Input Option for EC2

.github/workflows/python-ec2-asg-test.yml

@@ -14,6 +14,11 @@ on:
       caller-workflow-name:
         required: true
         type: string
+      python-version:
+        description: "Currently support version 3.8, 3.9, 3.10, 3.11, 3.12"
+        required: false
+        type: string
+        default: '3.10'
       staging-wheel-name:
         required: false
         default: 'aws-opentelemetry-distro'
@@ -31,8 +36,9 @@ permissions:
 env:
   E2E_TEST_AWS_REGION: ${{ inputs.aws-region }}
   CALLER_WORKFLOW_NAME: ${{ inputs.caller-workflow-name }}
+  PYTHON_VERSION: ${{ inputs.python-version }}
   ADOT_WHEEL_NAME: ${{ inputs.staging-wheel-name }}
-  SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/python-sample-app.zip
+  SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}-2/python-sample-app.zip
   E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
   E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
   METRIC_NAMESPACE: ApplicationSignals
@@ -56,7 +62,7 @@ jobs:
 
       - uses: actions/checkout@v4
         with:
-          repository: aws-observability/aws-application-signals-test-framework
+          repository: harrryr/aws-application-signals-test-framework
           ref: ${{ inputs.caller-workflow-name == 'main-build' && 'main' || github.ref }}
           fetch-depth: 0
 
@@ -144,6 +150,7 @@ jobs:
               -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \
               -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \
               -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \
+              -var="language_version=${{ env.PYTHON_VERSION }}" \
             || deployment_failed=$?
           
             if [ $deployment_failed -eq 1 ]; then

.github/workflows/python-ec2-canary.yml

@@ -29,11 +29,13 @@ jobs:
     with:
       aws-region: ${{ matrix.aws-region }}
       caller-workflow-name: 'appsignals-python-e2e-ec2-canary-test'
+      python-version: '3.10'
 
   pypi:
     uses: ./.github/workflows/python-ec2-default-retry.yml
     secrets: inherit
     with:
       aws-region: 'us-east-1'
       caller-workflow-name: 'appsignals-python-e2e-ec2-pypi-canary-test'
+      python-version: '3.10'
       otel-source: 'pypi'

.github/workflows/python-ec2-default-retry.yml

@@ -14,6 +14,9 @@ on:
       caller-workflow-name:
         required: true
         type: string
+      python-version:
+        required: true
+        type: string
       staging-wheel-name:
         required: false
         default: 'aws-opentelemetry-distro'
@@ -34,6 +37,7 @@ jobs:
     with:
       aws-region: ${{ inputs.aws-region }}
       caller-workflow-name: ${{ inputs.caller-workflow-name }}
+      python-version: ${{ inputs.python-version }}
       staging-wheel-name: ${{ inputs.staging-wheel-name }}
       otel-source: ${{ inputs.otel-source }}
 
@@ -45,6 +49,7 @@ jobs:
     with:
       aws-region: ${{ inputs.aws-region }}
       caller-workflow-name: ${{ inputs.caller-workflow-name }}
+      python-version: ${{ inputs.python-version }}
       staging-wheel-name: ${{ inputs.staging-wheel-name }}
       otel-source: ${{ inputs.otel-source }}
 

.github/workflows/python-ec2-default-test.yml

@@ -14,6 +14,11 @@ on:
       caller-workflow-name:
         required: true
         type: string
+      python-version:
+        description: "Currently support version 3.8, 3.9, 3.10, 3.11, 3.12"
+        required: false
+        type: string
+        default: '3.10'
       staging-wheel-name:
         required: false
         default: 'aws-opentelemetry-distro'
@@ -35,9 +40,10 @@ permissions:
 env:
   E2E_TEST_AWS_REGION: ${{ inputs.aws-region }}
   CALLER_WORKFLOW_NAME: ${{ inputs.caller-workflow-name }}
+  PYTHON_VERSION: ${{ inputs.python-version }}
   ADOT_WHEEL_NAME: ${{ inputs.staging-wheel-name }}
   OTEL_SOURCE: ${{ inputs.otel-source }}
-  SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/python-sample-app.zip
+  SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}-2/python-sample-app.zip
   E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
   E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
   METRIC_NAMESPACE: ApplicationSignals
@@ -58,21 +64,21 @@ jobs:
 
       - uses: actions/checkout@v4
         with:
-          repository: 'aws-observability/aws-application-signals-test-framework'
+          repository: 'harrryr/aws-application-signals-test-framework'
           ref: ${{ inputs.caller-workflow-name == 'main-build' && 'main' || github.ref }}
           fetch-depth: 0
 
         # We initialize Gradlew Daemon early on during the workflow because sometimes initialization
         # fails due to transient issues. If it fails here, then we will try again later before the validators
-      - name: Initiate Gradlew Daemon
-        id: initiate-gradlew
-        uses: ./.github/workflows/actions/execute_and_retry
-        continue-on-error: true
-        with:
-          command: "./gradlew :validator:build"
-          cleanup: "./gradlew clean"
-          max_retry: 3
-          sleep_time: 60
+#      - name: Initiate Gradlew Daemon
+#        id: initiate-gradlew
+#        uses: ./.github/workflows/actions/execute_and_retry
+#        continue-on-error: true
+#        with:
+#          command: "./gradlew :validator:build"
+#          cleanup: "./gradlew clean"
+#          max_retry: 3
+#          sleep_time: 60
 
       - name: Generate testing id
         run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}-${RANDOM}" >> $GITHUB_ENV
@@ -100,14 +106,14 @@ jobs:
         run: |
           if [ "${{ github.event.repository.name }}" = "aws-otel-python-instrumentation" ]; then
             # Reusing the adot-main-build-staging-jar bucket to store the python wheel file
-            echo GET_ADOT_WHEEL_COMMAND="aws s3 cp s3://adot-main-build-staging-jar/${{ env.ADOT_WHEEL_NAME }} ./${{ env.ADOT_WHEEL_NAME }} && python3.9 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
+            echo GET_ADOT_WHEEL_COMMAND="aws s3 cp s3://adot-main-build-staging-jar/${{ env.ADOT_WHEEL_NAME }} ./${{ env.ADOT_WHEEL_NAME }} && python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
           elif [ "${{ env.OTEL_SOURCE }}" == "pypi" ]; then
-            echo GET_ADOT_WHEEL_COMMAND="python3.9 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
+            echo GET_ADOT_WHEEL_COMMAND="python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
           else
             latest_release_version=$(curl -sL https://github.com/aws-observability/aws-otel-python-instrumentation/releases/latest | grep -oP '/releases/tag/v\K[0-9]+\.[0-9]+\.[0-9]+' | head -n 1)
             echo "The latest version is $latest_release_version"
             echo GET_ADOT_WHEEL_COMMAND="wget -O ${{ env.ADOT_WHEEL_NAME }} https://github.com/aws-observability/aws-otel-python-instrumentation/releases/latest/download/aws_opentelemetry_distro-$latest_release_version-py3-none-any.whl \
-            && python3.9 -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
+            && python${{ env.PYTHON_VERSION }} -m pip install ${{ env.ADOT_WHEEL_NAME }}" >> $GITHUB_ENV
           fi
 
       - name: Set Get CW Agent command environment variable
@@ -153,6 +159,7 @@ jobs:
               -var="sample_app_zip=${{ env.SAMPLE_APP_ZIP }}" \
               -var="get_cw_agent_rpm_command=${{ env.GET_CW_AGENT_RPM_COMMAND }}" \
               -var="get_adot_wheel_command=${{ env.GET_ADOT_WHEEL_COMMAND }}" \
+              -var="language_version=${{ env.PYTHON_VERSION }}" \
             || deployment_failed=$?
           
             if [ $deployment_failed -eq 1 ]; then
@@ -191,7 +198,6 @@ jobs:
         working-directory: terraform/python/ec2/default
 
       - name: Initiate Gradlew Daemon
-        if: steps.initiate-gradlew == 'failure'
         uses: ./.github/workflows/actions/execute_and_retry
         continue-on-error: true
         with:

.github/workflows/test-2.yml

@@ -0,0 +1,91 @@
+## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+## SPDX-License-Identifier: Apache-2.0
+
+## This workflow aims to run the Application Signals Python end-to-end tests as a canary to
+## test the artifacts for Application Signals enablement. It will deploy a sample app and remote
+## service on two EC2 instances, call the APIs, and validate the generated telemetry,
+## including logs, metrics, and traces.
+name: Python EC2 Enablement Canary Testing
+on:
+  push:
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  github-3-8:
+    strategy:
+      fail-fast: false
+      matrix:
+        aws-region: ['us-east-1']
+    uses: ./.github/workflows/python-ec2-default-retry.yml
+    secrets: inherit
+    with:
+      aws-region: ${{ matrix.aws-region }}
+      caller-workflow-name: 'appsignals-python-e2e-ec2-canary-test'
+      python-version: '3.8'
+
+  github-3-9:
+    strategy:
+      fail-fast: false
+      matrix:
+        aws-region: ['us-east-1']
+    uses: ./.github/workflows/python-ec2-default-retry.yml
+    secrets: inherit
+    with:
+      aws-region: ${{ matrix.aws-region }}
+      caller-workflow-name: 'appsignals-python-e2e-ec2-canary-test'
+      python-version: '3.9'
+  github-3-10:
+    strategy:
+      fail-fast: false
+      matrix:
+        aws-region: ['us-east-1']
+    uses: ./.github/workflows/python-ec2-default-retry.yml
+    secrets: inherit
+    with:
+      aws-region: ${{ matrix.aws-region }}
+      caller-workflow-name: 'appsignals-python-e2e-ec2-canary-test'
+      python-version: '3.10'
+
+  github-3-11:
+    strategy:
+      fail-fast: false
+      matrix:
+        aws-region: ['us-east-1']
+    uses: ./.github/workflows/python-ec2-default-retry.yml
+    secrets: inherit
+    with:
+      aws-region: ${{ matrix.aws-region }}
+      caller-workflow-name: 'appsignals-python-e2e-ec2-canary-test'
+      python-version: '3.11'
+
+  github-3-12:
+    strategy:
+      fail-fast: false
+      matrix:
+        aws-region: ['us-east-1']
+    uses: ./.github/workflows/python-ec2-default-retry.yml
+    secrets: inherit
+    with:
+      aws-region: ${{ matrix.aws-region }}
+      caller-workflow-name: 'appsignals-python-e2e-ec2-canary-test'
+      python-version: '3.12'
+
+  pypi:
+    uses: ./.github/workflows/python-ec2-default-retry.yml
+    secrets: inherit
+    with:
+      aws-region: 'us-east-1'
+      caller-workflow-name: 'appsignals-python-e2e-ec2-pypi-canary-test'
+      python-version: '3.9'
+      otel-source: 'pypi'
+
+  asg:
+    uses: ./.github/workflows/python-ec2-asg-test.yml
+    secrets: inherit
+    with:
+      aws-region: 'us-east-1'
+      caller-workflow-name: 'appsignals-python-e2e-ec2-pypi-canary-test'
+      python-version: '3.9'

sample-apps/python/django_frontend_service/ec2-requirements.txt

@@ -1,6 +1,8 @@
 Django~=4.2.9
-boto3~=1.34.3
+boto3~=1.34.161
 pymysql==1.1.1
 python-dotenv~=1.0.1
 requests~=2.31.0
 schedule~=1.2.1
+opentelemetry-sdk=~1.27.0
+opentelemetry-api~=1.27.0

terraform/python/ec2/asg/main.tf

@@ -98,8 +98,32 @@ resource "aws_launch_configuration" "launch_configuration" {
     # Install Python and wget
     sudo yum install wget -y
     sudo yum install unzip -y
-    sudo dnf install -y python3.9
-    sudo dnf install -y python3.9-pip
+
+      # Dnf does not have the module for python 3.8, 3,10, 3.12, therefore we need to manually install it by downloading the package from the python website.
+      # Building and installing the package takes longer then installing it through dnf, so a seperate installation process was made.
+      # The canary should run on a version without the manual installation process
+      if [ "${var.language_version}" == "3.8" ] || [ "${var.language_version}" = "3.10" ] || [ "${var.language_version}" = "3.12" ]; then
+          # Install modules required to compile Python and also run the sample app
+          sudo dnf groupinstall "Development Tools" -y
+          sudo dnf install openssl-devel sqlite-devel libffi-devel -y
+
+          # Download the Python package
+          cd /usr/src
+          sudo wget https://www.python.org/ftp/python/${var.language_version}.0/Python-${var.language_version}.0.tgz
+          sudo tar xzf Python-${var.language_version}.0.tgz
+
+          # Compile and install Python using c++
+          cd Python-${var.language_version}.0
+          sudo ./configure --with-openssl
+          sudo ./configure --enable-optimizations
+          sudo make altinstall
+
+          # Return back to ec2-user directory
+          cd /home/ec2-user
+      else
+        sudo dnf install -y python${var.language_version}
+        sudo dnf install -y python${var.language_version}-pip
+      fi
 
     # Copy in CW Agent configuration
     agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
@@ -122,7 +146,7 @@ resource "aws_launch_configuration" "launch_configuration" {
     # Delete the requests requirement as it is installed already using rpm. Only applicable for ec2 instances
     # created by Auto Scaling Groups
     sudo sed -i '/requests/d' ./ec2-requirements.txt
-    python3.9 -m pip install -r ec2-requirements.txt
+    python${var.language_version} -m pip install -r ec2-requirements.txt
     export DJANGO_SETTINGS_MODULE="django_frontend_service.settings"
     export OTEL_PYTHON_DISTRO="aws_distro"
     export OTEL_PYTHON_CONFIGURATOR="aws_configurator"
@@ -135,8 +159,8 @@ resource "aws_launch_configuration" "launch_configuration" {
     export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc
     export OTEL_SERVICE_NAME=python-sample-application-${var.test_id}
     export OTEL_TRACES_SAMPLER=always_on
-    python3.9 manage.py migrate
-    nohup opentelemetry-instrument python3.9 manage.py runserver 0.0.0.0:8000 --noreload &
+    python${var.language_version} manage.py migrate
+    nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8000 --noreload &
 
     # The application needs time to come up and reach a steady state, this should not take longer than 30 seconds
     sleep 30
@@ -145,7 +169,7 @@ resource "aws_launch_configuration" "launch_configuration" {
     sudo yum install nodejs aws-cli unzip tmux -y
 
     # Bring in the traffic generator files to EC2 Instance
-    aws s3 cp s3://aws-appsignals-sample-app-prod-${var.aws_region}/traffic-generator.zip ./traffic-generator.zip
+    aws s3 cp s3://aws-appsignals-sample-app-prod-${var.aws_region}-2/traffic-generator.zip ./traffic-generator.zip
     unzip ./traffic-generator.zip -d ./
 
     # Install the traffic generator dependencies
@@ -203,8 +227,32 @@ resource "null_resource" "remote_service_setup" {
       # Install Python and wget
       sudo yum install wget -y
       sudo yum install unzip -y
-      sudo dnf install -y python3.9
-      sudo dnf install -y python3.9-pip
+
+      # Dnf does not have the module for python 3.8, 3,10, 3.12, therefore we need to manually install it by downloading the package from the python website.
+      # Building and installing the package takes longer then installing it through dnf, so a seperate installation process was made.
+      # The canary should run on a version without the manual installation process
+      if [ "${var.language_version}" == "3.8" ] || [ "${var.language_version}" = "3.10" ] || [ "${var.language_version}" = "3.12" ]; then
+          # Install modules required to compile Python and also run the sample app
+          sudo dnf groupinstall "Development Tools" -y
+          sudo dnf install openssl-devel sqlite-devel libffi-devel -y
+
+          # Download the Python package
+          cd /usr/src
+          sudo wget https://www.python.org/ftp/python/${var.language_version}.0/Python-${var.language_version}.0.tgz
+          sudo tar xzf Python-${var.language_version}.0.tgz
+
+          # Compile and install Python using c++
+          cd Python-${var.language_version}.0
+          sudo ./configure --with-openssl
+          sudo ./configure --enable-optimizations
+          sudo make altinstall
+
+          # Return back to ec2-user directory
+          cd /home/ec2-user
+      else
+        sudo dnf install -y python${var.language_version}
+        sudo dnf install -y python${var.language_version}-pip
+      fi
 
       # Copy in CW Agent configuration
       agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
@@ -225,7 +273,7 @@ resource "null_resource" "remote_service_setup" {
       # Export environment variables for instrumentation
       cd ./django_remote_service
       export DJANGO_SETTINGS_MODULE="django_remote_service.settings"
-      python3.9 -m pip install -r requirements.txt --force-reinstall
+      python${var.language_version} -m pip install -r requirements.txt --force-reinstall
       export OTEL_PYTHON_DISTRO="aws_distro"
       export OTEL_PYTHON_CONFIGURATOR="aws_configurator"
       export OTEL_METRICS_EXPORTER=none
@@ -237,8 +285,8 @@ resource "null_resource" "remote_service_setup" {
       export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc
       export OTEL_SERVICE_NAME=python-sample-remote-application-${var.test_id}
       export OTEL_TRACES_SAMPLER=always_on
-      python3.9 manage.py migrate
-      nohup opentelemetry-instrument python3.9 manage.py runserver 0.0.0.0:8001 --noreload &
+      python${var.language_version} manage.py migrate
+      nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8001 --noreload &
 
       # The application needs time to come up and reach a steady state, this should not take longer than 30 seconds
       sleep 30