From 2ebece360f7cbc051856cce6f45175958da7a6b6 Mon Sep 17 00:00:00 2001 From: Michael Fu <42988210+michaelfu1029@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:29:50 -0700 Subject: [PATCH] Turn off scarf based telemetry data collection (#149) *Issue #, if available:* *Description of changes:* Turning off scarf based telemetry https://airflow.apache.org/docs/apache-airflow/2.10.1/configurations-ref.html#usage-data-collection Basically adding an extra environment variable for this Also updated readme to show 2.10.1 support By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. --------- Co-authored-by: Michael Fu --- README.md | 7 +++---- .../airflow/2.10.1/python/mwaa/config/airflow.py | 16 +++++++++++++++- .../airflow/2.9.2/python/mwaa/config/airflow.py | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 401169c..fc97aae 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/) uses to run You can also use it locally if you want to run a MWAA-like environment for testing, experimentation, and development purposes. -Currently, Airflow v2.9.2 is supported. Future versions in parity with Amazon MWAA will be added as +Currently, Airflow v2.9.2 and v2.10.1 are supported. Future versions in parity with Amazon MWAA will be added as well. _Notice, however, that we do not plan to support previous Airflow versions supported by MWAA._ ## Using the Airflow Image @@ -26,7 +26,7 @@ To experiment with the image using a vanilla Docker setup, follow these steps: python3 create_venvs.py --target ``` -3. Build the Airflow v2.9.2 Docker image +3. Build a supported Airflow version Docker image - `cd /images/airflow/2.9.2` - Update `run.sh` file with your account ID, environment name and account credentials. The permissions associated with the provided credentials will be assigned to the Airflow components that would be started with the next step. @@ -46,8 +46,7 @@ Airflow should be up and running now. You can access the web server on your loca When you build the Docker images of a certain Airflow version, using either `build.sh` or `run.sh` (which automatically also calls `build.sh` for you), multiple Docker images will actually be -generated. For example, for Airflow 2.9 (the only currently supported version), you will notice the -following images: +generated. For example, for Airflow 2.9, you will notice the following images: | Repository | Tag | | --------------------------------- | ----------------------------- | diff --git a/images/airflow/2.10.1/python/mwaa/config/airflow.py b/images/airflow/2.10.1/python/mwaa/config/airflow.py index 0b47663..68222d7 100644 --- a/images/airflow/2.10.1/python/mwaa/config/airflow.py +++ b/images/airflow/2.10.1/python/mwaa/config/airflow.py @@ -242,6 +242,19 @@ def _get_opinionated_airflow_secrets_config() -> Dict[str, str]: "AIRFLOW__SECRETS__BACKEND_KWARGS": json.dumps(connection_lookup_pattern), } +def _get_opinionated_airflow_usage_data_config() -> Dict[str, str]: + """ + Retrieve the environment variables for Airflow's usage data configuration section. + + This config can be overridden by the user. + + :returns A dictionary containing the environment variables. + """ + + return { + "AIRFLOW__USAGE_DATA_COLLECTION__ENABLED": "False", + } + def _get_essential_airflow_webserver_config() -> Dict[str, str]: """ @@ -310,7 +323,7 @@ def get_opinionated_airflow_config() -> Dict[str, str]: """ Retrieve the environment variables required to set Airflow configurations. - These environment variables are essential and cannot be overridden by the customer. + These environment variables can be overridden by the customer. :returns A dictionary containing the environment variables. """ @@ -318,4 +331,5 @@ def get_opinionated_airflow_config() -> Dict[str, str]: **_get_opinionated_airflow_core_config(), **_get_opinionated_airflow_scheduler_config(), **_get_opinionated_airflow_secrets_config(), + **_get_opinionated_airflow_usage_data_config(), } diff --git a/images/airflow/2.9.2/python/mwaa/config/airflow.py b/images/airflow/2.9.2/python/mwaa/config/airflow.py index 0b47663..f69e5bb 100644 --- a/images/airflow/2.9.2/python/mwaa/config/airflow.py +++ b/images/airflow/2.9.2/python/mwaa/config/airflow.py @@ -310,7 +310,7 @@ def get_opinionated_airflow_config() -> Dict[str, str]: """ Retrieve the environment variables required to set Airflow configurations. - These environment variables are essential and cannot be overridden by the customer. + These environment variables can be overridden by the customer. :returns A dictionary containing the environment variables. """