Support aws opentelemetry distro in Lambda Python Runtimes (#241)

*Issue #, if available:*

*Description of changes:*

Support aws opentelemetry distro in AWS Lambda Python Runtimes. The PR
contains:

* Build aws opentelemetry distro Lambda python layer
Run `build-lambda-layer.sh` in
`aws-otel-python-instrumentation/lambda-layer/src`

* Unit test for aws opentelemetry distro Lambda python layer
Run `tox` in `aws-otel-python-instrumentation/lambda-layer/src`

* Deploy the aws opentelemetry distro Lambda python layer with an
Application Signals enabled Sample App in personal AWS account.
Run `build.sh` in `aws-otel-python-instrumentation/lambda-layer`

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -250,7 +250,8 @@ def _customize_exporter(span_exporter: SpanExporter, resource: Resource) -> Span
     if not _is_application_signals_enabled():
         return span_exporter
     if _is_lambda_environment():
-        return AwsMetricAttributesSpanExporterBuilder(OTLPUdpSpanExporter(), resource).build()
+        traces_endpoint = os.environ.get(AWS_XRAY_DAEMON_ADDRESS_CONFIG, "127.0.0.1:2000")
+        return AwsMetricAttributesSpanExporterBuilder(OTLPUdpSpanExporter(endpoint=traces_endpoint), resource).build()
     return AwsMetricAttributesSpanExporterBuilder(span_exporter, resource).build()
 
 

lambda-layer/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+pushd src || exit
+rm -rf build
+./build-lambda-layer.sh
+popd || exit
+
+pushd sample-apps || exit
+rm -rf build
+./package-lambda-function.sh
+popd || exit
+
+pushd terraform/lambda || exit
+terraform init
+terraform apply -auto-approve
+popd || exit

lambda-layer/sample-apps/function/lambda_function.py

@@ -0,0 +1,17 @@
+import json
+import os
+
+import boto3
+import requests
+
+client = boto3.client("s3")
+
+
+# lambda function
+def lambda_handler(event, context):
+
+    requests.get("https://aws.amazon.com/")
+
+    client.list_buckets()
+
+    return {"statusCode": 200, "body": json.dumps(os.environ.get("_X_AMZN_TRACE_ID"))}

lambda-layer/sample-apps/function/requirements.txt

@@ -0,0 +1 @@
+requests

lambda-layer/sample-apps/package-lambda-function.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+mkdir -p build/python
+python3 -m pip install -r function/requirements.txt -t build/python
+cp function/lambda_function.py build/python
+cd build/python
+zip -r ../function.zip ./*

lambda-layer/src/Dockerfile

@@ -0,0 +1,19 @@
+ARG runtime=python3.12
+
+FROM public.ecr.aws/sam/build-${runtime}
+
+ADD . /workspace
+
+WORKDIR /workspace
+
+RUN mkdir -p /build && \
+  python3 -m pip install aws-opentelemetry-distro/ -t /build/python && \
+  mv otel_wrapper.py /build/python && \
+  mv otel-instrument /build && \
+  chmod 755 /build/otel-instrument && \
+  rm -rf /build/python/boto* && \
+  rm -rf /build/python/urllib3* && \
+  cd /build && \
+  zip -r aws-opentelemetry-python-layer.zip otel-instrument python
+
+CMD ["cp", "/build/aws-opentelemetry-python-layer.zip", "/out/aws-opentelemetry-python-layer.zip"]

lambda-layer/src/build-lambda-layer.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -e
+
+rm -rf build
+rm -rf ./aws-opentelemetry-distro
+cp -r ../../aws-opentelemetry-distro ./
+mkdir -p build
+docker build --progress plain -t aws-opentelemetry-python-layer .
+docker run --rm -v "$(pwd)/build:/out" aws-opentelemetry-python-layer
+rm -rf ./aws-opentelemetry-distro

lambda-layer/src/otel-instrument

@@ -0,0 +1,146 @@
+#!/bin/bash
+
+set -ef -o pipefail
+
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+: <<'END_DOCUMENTATION'
+`otel-instrument`
+
+This script configures and sets up OpenTelemetry Python with the values we
+expect will be used by the common user. It does this by setting the environment
+variables OpenTelemetry uses, and then initializing OpenTelemetry using the
+`opentelemetry-instrument` auto instrumentation script from the
+`opentelemetry-instrumentation` package.
+
+Additionally, this configuration assumes the user is using packages conforming
+to the `opentelemetry-instrumentation` and `opentelemetry-sdk` specifications.
+
+DO NOT use this script for anything else besides SETTING ENVIRONMENT VARIABLES.
+
+See more:
+https://docs.aws.amazon.com/lambda/latest/dg/runtimes-modify.html#runtime-wrapper
+
+Usage
+-----
+We expect this file to be at the root of a Lambda Layer. Having it anywhere else
+seems to mean AWS Lambda cannot find it.
+
+In the configuration of an AWS Lambda function with this file at the
+root level of a Lambda Layer:
+
+.. code::
+
+    AWS_LAMBDA_EXEC_WRAPPER = /opt/otel-instrument
+
+END_DOCUMENTATION
+
+# Use constants to access the environment variables we want to use in this
+# script.
+
+# See more:
+# https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html#configuration-envvars-runtime
+
+# - Reserved environment variables
+
+# - - $AWS_LAMBDA_FUNCTION_NAME
+# - - $LAMBDA_RUNTIME_DIR
+
+# - Unreserved environment variables
+
+# - - $PYTHONPATH
+
+# Update the python paths for packages with `sys.path` and `PYTHONPATH`
+
+# - We know that the path to the Lambda Layer OpenTelemetry Python packages are
+#   well defined, so we can add them to the PYTHONPATH.
+#
+#   See more:
+#   https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path
+
+export LAMBDA_LAYER_PKGS_DIR="/opt/python";
+
+# - Set Lambda Layer python packages in PYTHONPATH so `opentelemetry-instrument`
+#   script can find them (it needs to find `opentelemetry` to find the auto
+#   instrumentation `run()` method later)
+
+export PYTHONPATH="$LAMBDA_LAYER_PKGS_DIR:$PYTHONPATH";
+
+# - Set Lambda runtime python packages in PYTHONPATH so
+#   `opentelemetry-instrument` script can find them during auto instrumentation
+#   and instrument them.
+
+export PYTHONPATH="$LAMBDA_RUNTIME_DIR:$PYTHONPATH";
+
+# Configure OpenTelemetry Python with environment variables
+
+# - We leave `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` to its default. This is
+#   `http://localhost:4318/v1/traces` because we are using the HTTP exporter
+
+# - If OTEL_EXPORTER_OTLP_PROTOCOL is not set by user, the default exporting protocol is http/protobuf.
+if [ -z "${OTEL_EXPORTER_OTLP_PROTOCOL}" ]; then
+    export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+fi
+
+# - Set the service name
+
+if [ -z "${OTEL_SERVICE_NAME}" ]; then
+    export OTEL_SERVICE_NAME=$AWS_LAMBDA_FUNCTION_NAME;
+fi
+
+# - Set the propagators
+
+if [[ -z "$OTEL_PROPAGATORS" ]]; then
+  export OTEL_PROPAGATORS="tracecontext,baggage,xray"
+fi
+
+export LAMBDA_RESOURCE_ATTRIBUTES="cloud.region=$AWS_REGION,cloud.provider=aws,faas.name=$AWS_LAMBDA_FUNCTION_NAME,faas.version=$AWS_LAMBDA_FUNCTION_VERSION,faas.instance=$AWS_LAMBDA_LOG_STREAM_NAME,aws.log.group.names=$AWS_LAMBDA_LOG_GROUP_NAME";
+
+# - If Application Signals is enabled
+
+if [ "${OTEL_AWS_APPLICATION_SIGNALS_ENABLED}" = "true" ]; then
+    export OTEL_PYTHON_DISTRO="aws_distro";
+    export OTEL_PYTHON_CONFIGURATOR="aws_configurator";
+    export OTEL_METRICS_EXPORTER="none";
+    export OTEL_LOGS_EXPORTER="none";
+fi
+
+if [ -z "${OTEL_RESOURCE_ATTRIBUTES}" ]; then
+    export OTEL_RESOURCE_ATTRIBUTES=$LAMBDA_RESOURCE_ATTRIBUTES;
+else
+    export OTEL_RESOURCE_ATTRIBUTES="$LAMBDA_RESOURCE_ATTRIBUTES,$OTEL_RESOURCE_ATTRIBUTES";
+fi
+
+# - Enable botocore instrumentation by default
+
+if [ -z ${OTEL_PYTHON_DISABLED_INSTRUMENTATIONS} ]; then
+    export OTEL_PYTHON_DISABLED_INSTRUMENTATIONS="aio-pika,aiohttp-client,aiopg,asgi,asyncpg,boto3sqs,boto,cassandra,celery,confluent-kafka,dbapi,django,elasticsearch,fastapi,falcon,flask,grpc,httpx,jinja2,kafka-python,logging,mysql,mysqlclient,pika,psycopg2,pymemcache,pymongo,pymysql,pyramid,redis,remoulade,requests,sklearn,sqlalchemy,sqlite3,starlette,system-metrics,tornado,tortoiseorm,urllib,urllib3,wsgi"
+fi
+
+# - Use a wrapper because AWS Lambda's `python3 /var/runtime/bootstrap.py` will
+#   use `imp.load_module` to load the function from the `_HANDLER` environment
+#   variable. This RELOADS the module and REMOVES any instrumentation patching
+#   done earlier. So we delay instrumentation until `bootstrap.py` imports
+#   `otel_wrapper.py` at which we know the patching will be picked up.
+#
+#   See more:
+#   https://docs.python.org/3/library/imp.html#imp.load_module
+
+export ORIG_HANDLER=$_HANDLER;
+export _HANDLER="otel_wrapper.lambda_handler";
+
+# - Call the upstream auto instrumentation script
+
+exec python3 $LAMBDA_LAYER_PKGS_DIR/bin/opentelemetry-instrument "$@"

lambda-layer/src/otel_wrapper.py

@@ -0,0 +1,66 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+`otel_wrapper.py`
+
+This file serves as a wrapper over the user's Lambda function.
+
+Usage
+-----
+Patch the reserved `_HANDLER` Lambda environment variable to point to this
+file's `otel_wrapper.lambda_handler` property. Do this having saved the original
+`_HANDLER` in the `ORIG_HANDLER` environment variable. Doing this makes it so
+that **on import of this file, the handler is instrumented**.
+
+Instrumenting any earlier will cause the instrumentation to be lost because the
+AWS Service uses `imp.load_module` to import the handler which RELOADS the
+module. This is why AwsLambdaInstrumentor cannot be instrumented with the
+`opentelemetry-instrument` script.
+
+See more:
+https://docs.python.org/3/library/imp.html#imp.load_module
+
+"""
+
+import os
+from importlib import import_module
+
+from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
+
+
+def modify_module_name(module_name):
+    """Returns a valid modified module to get imported"""
+    return ".".join(module_name.split("/"))
+
+
+class HandlerError(Exception):
+    pass
+
+
+AwsLambdaInstrumentor().instrument()
+
+path = os.environ.get("ORIG_HANDLER")
+
+if path is None:
+    raise HandlerError("ORIG_HANDLER is not defined.")
+
+try:
+    (mod_name, handler_name) = path.rsplit(".", 1)
+except ValueError as e:
+    raise HandlerError("Bad path '{}' for ORIG_HANDLER: {}".format(path, str(e)))
+
+modified_mod_name = modify_module_name(mod_name)
+handler_module = import_module(modified_mod_name)
+lambda_handler = getattr(handler_module, handler_name)

lambda-layer/src/tests/mocks/lambda_function.py

@@ -0,0 +1,17 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def handler(event, context):
+    return "200 ok"

lambda-layer/src/tests/requirements.txt

@@ -0,0 +1,3 @@
+# Dependencies used in tests only
+opentelemetry-test-utils==0.46b0
+opentelemetry-instrumentation-aws-lambda==0.46b0