From 722af3449779bc859dc35ecc4d45e739b3b02d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Fri, 6 Oct 2023 13:27:29 -0300 Subject: [PATCH] refactor(airflow): refactor airflow containers (#200) * refactor(airflow): refactor airflow containers * use compose-go instead of docker-compose (conda) * Add config for airflow version * configure executor to use postgres connection * Include python environments on airflow containers * install pyenvs via requirements.txt * owid DAG * Include EGH args on dockerfile to create DB connection config on airflow docker image * Finish OWID DAG * Update colombia DAG * Trying to send information through external tasks * remove the external in which was blocking the creation of other tasks, use requests instead * Finish FOPH metadata DAG * remove unnecessary env template --- .containers-sugar.yaml | 18 + .env.tpl | 2 + .github/workflows/main.yaml | 2 + conda/airflow.yaml | 6 +- conda/base.yaml | 3 +- containers/airflow/Dockerfile | 164 +- containers/airflow/README.md | 9 + containers/airflow/airflow.cfg | 1234 ---------- containers/airflow/config/airflow.cfg | 2080 +++++++++++++++++ containers/airflow/dags/brasil/sinan_ctl.py | 113 - .../airflow/dags/brasil/sinan_drop_table.py | 52 - containers/airflow/dags/colombia_dag.py | 32 +- containers/airflow/dags/owid_dag.py | 47 +- .../airflow/dags/switzerland/foph_dag.py | 7 +- .../dags/switzerland/foph_metadata_dag.py | 177 +- .../dags/switzerland/foph_weekly_dag.py | 94 +- .../webserver_config.py => envs/README.md} | 0 containers/airflow/envs/epigraphhub.txt | 1 + containers/airflow/envs/pysus.txt | 1 + containers/airflow/scripts/entrypoint.sh | 341 ++- containers/airflow/scripts/init-db.sh | 48 - containers/airflow/scripts/startup.sh | 2 +- containers/compose-airflow.yaml | 243 ++ 23 files changed, 2977 insertions(+), 1699 deletions(-) create mode 100644 .containers-sugar.yaml create mode 100644 containers/airflow/README.md delete mode 100644 containers/airflow/airflow.cfg create mode 100644 containers/airflow/config/airflow.cfg delete mode 100644 containers/airflow/dags/brasil/sinan_ctl.py delete mode 100644 containers/airflow/dags/brasil/sinan_drop_table.py rename containers/airflow/{scripts/webserver_config.py => envs/README.md} (100%) create mode 100644 containers/airflow/envs/epigraphhub.txt create mode 100644 containers/airflow/envs/pysus.txt delete mode 100755 containers/airflow/scripts/init-db.sh create mode 100644 containers/compose-airflow.yaml diff --git a/.containers-sugar.yaml b/.containers-sugar.yaml new file mode 100644 index 00000000..dfe90fc1 --- /dev/null +++ b/.containers-sugar.yaml @@ -0,0 +1,18 @@ +version: 1.9.0 +compose-app: docker-compose +env-file: .env + +service-groups: + - name: airflow + project-name: egh-airflow + compose-path: + - containers/compose-airflow.yaml + env-file: containers/airflow/.env + services: + default: webserver,scheduler,worker,triggerer + available: + - name: webserver + - name: scheduler + - name: worker + - name: triggerer + - name: airflow-cli diff --git a/.env.tpl b/.env.tpl index 4b10327c..23c6945a 100644 --- a/.env.tpl +++ b/.env.tpl @@ -42,6 +42,8 @@ POSTGRES_PASSWORD=${POSTGRES_PASSWORD} POSTGRES_DB=${POSTGRES_DB} POSTGRES_DATA_DIR_HOST=${POSTGRES_DATA_DIR_HOST} POSTGRES_CONFIG_FILE_HOST=${POSTGRES_CONFIG_FILE_HOST} +POSTGRES_EPIGRAPH_HOST=${POSTGRES_EPIGRAPH_HOST} +POSTGRES_EPIGRAPH_PORT=${POSTGRES_EPIGRAPH_PORT} POSTGRES_EPIGRAPH_USER=${POSTGRES_EPIGRAPH_USER} POSTGRES_EPIGRAPH_PASSWORD=${POSTGRES_EPIGRAPH_PASSWORD} POSTGRES_EPIGRAPH_DB=${POSTGRES_EPIGRAPH_DB} diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2b97b44c..74727492 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -39,6 +39,8 @@ env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres + POSTGRES_EPIGRAPH_HOST: postgres + POSTGRES_EPIGRAPH_PORT: 25432 POSTGRES_EPIGRAPH_USER: dev_epigraph POSTGRES_EPIGRAPH_PASSWORD: dev_epigraph POSTGRES_EPIGRAPH_DB: dev_epigraphhub diff --git a/conda/airflow.yaml b/conda/airflow.yaml index d9940b24..c8d1a731 100644 --- a/conda/airflow.yaml +++ b/conda/airflow.yaml @@ -1,9 +1,11 @@ +# Note: these dependencies are for dev only. to work on container, they have +# to be in an virtual environment to run in an isolated python version name: epigraphhub channels: - nodefaults - conda-forge dependencies: - - airflow 2.5.2 + - airflow 2.7.1 - fiona - geopandas - gsheetsdb @@ -23,5 +25,3 @@ dependencies: - pip - pip: - -r pip.txt - - epigraphhub - - pysus diff --git a/conda/base.yaml b/conda/base.yaml index 098b705d..e29e01a9 100644 --- a/conda/base.yaml +++ b/conda/base.yaml @@ -5,12 +5,13 @@ channels: dependencies: - python 3.9.* - awscli - - docker-compose - git - make - sqlite - webdriver-manager - pip - pip: + - containers-sugar + - compose-go - epigraphhub - "selenium<=4.0" diff --git a/containers/airflow/Dockerfile b/containers/airflow/Dockerfile index 02fbdaf8..96068ff4 100644 --- a/containers/airflow/Dockerfile +++ b/containers/airflow/Dockerfile @@ -1,6 +1,4 @@ -# ref: https://github.com/mamba-org/micromamba-docker/blob/main/Dockerfile - -FROM condaforge/mambaforge:latest +FROM apache/airflow:2.7.1 LABEL maintainer="Ivan Ogasawara " LABEL org.opencontainers.image.title="EpiGraphHub" @@ -13,15 +11,8 @@ LABEL org.thegraphnetwork.epigraphhub.version="latest" # it is the default, but using it here to have it explicitly USER root -SHELL ["/bin/bash", "-c"] -# Use bash in Dockerfile RUN commands and make sure bashrc is sourced when -# executing commands with /bin/bash -c -# Needed to have the micromamba activate command configured etc. - -ENV ENV_NAME=epigraphhub ENV DEBIAN_FRONTEND=noninteractive -ARG UID=1000 -ARG GID=1000 +ARG AIRFLOW_UID RUN apt-get update -y \ && apt-get install -y --no-install-recommends \ @@ -39,100 +30,85 @@ RUN apt-get update -y \ ca-certificates \ gnupg \ dirmngr \ - freetds-bin \ - freetds-dev \ - gosu \ - ldap-utils \ - libffi-dev \ - libpq-dev \ - libsasl2-2 \ - libsasl2-dev \ - libsasl2-modules \ - libssl-dev \ - locales \ - lsb-release \ - nodejs \ - openssh-client \ - postgresql-client \ - sasl2-bin \ - software-properties-common \ - sqlite3 \ - sudo \ - unixodbc \ - unixodbc-dev \ - yarn \ vim \ + libssl-dev \ + liblzo2-dev \ + libpam0g-dev \ + zlib1g-dev \ + libffi-dev \ + libbz2-dev \ + libsqlite3-dev \ && rm -rf /var/lib/apt/lists/* \ /var/cache/apt/archives \ - /tmp/* \ - && addgroup --gid ${GID} epigraphhub \ - && useradd --uid ${UID} --gid ${GID} -ms /bin/bash epigraphhub \ - && mkdir -p /opt/EpiGraphHub \ - && chmod -R a+rwx /opt/conda /opt/EpiGraphHub \ - && export ENV_NAME="$ENV_NAME" \ - && echo "epigraphhub ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/epigraphhub \ - && chmod 0440 /etc/sudoers.d/epigraphhub \ - && mkdir -p /opt/superset \ - && chown epigraphhub:epigraphhub /opt/superset \ - && chmod a+rw /var/log/ - -USER epigraphhub - -WORKDIR /opt/EpiGraphHub - -COPY --chown=epigraphhub:epigraphhub conda/ /tmp/conda - -ENV PATH /opt/conda/envs/$ENV_NAME/bin:$PATH -ENV PYTHONPATH='/opt/superset:/opt/EpiGraphHub' -ENV ANSIBLE_CONFIG='/opt/EpiGraphHub/playbooks/ansible.cfg' - -RUN mamba env create -n $ENV_NAME --file /tmp/conda/airflow.yaml \ - && conda clean --all \ - && find /opt/conda/ -type f,l -name '*.a' -delete \ - && find /opt/conda/ -type f,l -name '*.pyc' -delete \ - && find /opt/conda/ -type f,l -name '*.js.map' -delete \ - && rm -rf /opt/conda/pkgs /tmp/* - -# note: keeping it to the end of the recipes helps to avoid rebuilding the -# image after every change. -# COPY --chown=epigraphhub:epigraphhub . /opt/EpiGraphHub - -COPY --chown=epigraphhub:epigraphhub containers/superset/superset.sh /opt/superset.sh -# note: these files can be overwriten by docker compose volumes in order to -# use the last version without building the image again. -COPY --chown=epigraphhub:epigraphhub containers/superset/ /opt/superset -COPY --chown=epigraphhub:epigraphhub containers/superset/entrypoint.sh /opt/entrypoint.sh - -RUN chmod +x /opt/entrypoint.sh \ - && echo "source /opt/entrypoint.sh" > ~/.bashrc \ - && sudo mkdir -p /opt/data/superset/ \ - && sudo chown -R epigraphhub:epigraphhub /opt/data \ - && sudo chown -R epigraphhub:epigraphhub /var/log/* - -# note: the steps above were copied from the superset + some apt deps -# needed by airflow - -# ref: https://hub.docker.com/r/apache/airflow/dockerfile - -ENV AIRFLOW_HOME=/opt/airflow -ENV DEBIAN_FRONTEND=noninteractive + /tmp/* + +RUN usermod -u ${AIRFLOW_UID} -g 0 -d /home/airflow -s /bin/bash airflow \ + && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ + && chmod 0440 /etc/sudoers.d/airflow \ + && mkdir -p ${AIRFLOW_HOME}/scripts /opt/envs \ + && chown -R ${AIRFLOW_UID}:0 ${AIRFLOW_HOME} /opt/envs/ + +RUN curl https://www.python.org/ftp/python/3.10.8/Python-3.10.8.tgz -o /tmp/Python-3.10.8.tgz \ + && tar -zxvf /tmp/Python-3.10.8.tgz -C /tmp \ + && cd /tmp/Python-3.10.8 \ + && ./configure --prefix=/opt/py310 --enable-optimizations \ + && make install \ + && chown -R airflow /opt/py310 \ + && echo "alias python3.10=/opt/py310/bin/python3.10" >> /home/airflow/.bashrc \ + && rm -rf /tmp/Python-3.10* + +RUN curl https://www.python.org/ftp/python/3.11.6/Python-3.11.6.tgz -o /tmp/Python-3.11.6.tgz \ + && tar -zxvf /tmp/Python-3.11.6.tgz -C /tmp \ + && cd /tmp/Python-3.11.6 \ + && ./configure --prefix=/opt/py311 --enable-optimizations \ + && make install \ + && chown -R airflow /opt/py311 \ + && echo "alias python3.11=/opt/py311/bin/python3.11" >> /home/airflow/.bashrc \ + && rm -rf /tmp/Python-3.11* + +COPY --chown=airflow containers/airflow/config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg +COPY --chown=airflow containers/airflow/scripts/*.sh ${AIRFLOW_HOME}/scripts/ +COPY --chown=airflow containers/airflow/scripts/entrypoint.sh /opt/entrypoint.sh +COPY --chown=airflow containers/airflow/envs/* /opt/envs/ + +USER airflow + +ARG POSTGRES_EPIGRAPH_HOST +ARG POSTGRES_EPIGRAPH_PORT +ARG POSTGRES_EPIGRAPH_USER +ARG POSTGRES_EPIGRAPH_PASSWORD +ARG POSTGRES_EPIGRAPH_DB +ENV DB_USER "${POSTGRES_EPIGRAPH_USER}:${POSTGRES_EPIGRAPH_PASSWORD}" +ENV DB_URI "${DB_USER}@${POSTGRES_EPIGRAPH_HOST}:${POSTGRES_EPIGRAPH_PORT}/${POSTGRES_EPIGRAPH_DB}" + +RUN /usr/local/bin/python -m virtualenv /opt/envs/py310 --python="/opt/py310/bin/python3.10" \ + && sed -i "s/include-system-site-packages = false/include-system-site-packages = true/" /opt/envs/py310/pyvenv.cfg \ + && source /opt/envs/py310/bin/activate \ + && pip install "cython<3.0.0" \ + && pip install --no-build-isolation "pyyaml<6.0" \ + && pip install -r /opt/envs/epigraphhub.txt \ + && epigraphhub-config --name "epigraphhub" --db-uri "${DB_URI}" + +RUN /usr/local/bin/python -m virtualenv /opt/envs/py311 --python="/opt/py311/bin/python3.11" \ + && sed -i "s/include-system-site-packages = false/include-system-site-packages = true/" /opt/envs/py311/pyvenv.cfg \ + && source /opt/envs/py311/bin/activate \ + && pip install "cython<3.0.0" \ + && pip install --no-build-isolation "pyyaml<6.0" \ + && pip install -r /opt/envs/pysus.txt + +WORKDIR ${AIRFLOW_HOME} # ref: https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive RUN sudo ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime RUN sudo mkdir -p /opt/scripts /sources /opt/airflow \ - && sudo chown -R epigraphhub:epigraphhub /opt/scripts \ - && sudo chown -R epigraphhub:epigraphhub /sources \ - && sudo chown -R epigraphhub:epigraphhub /opt/airflow \ + && sudo chown -R airflow /opt/scripts \ + && sudo chown -R airflow /sources \ + && sudo chown -R airflow /opt/airflow \ && sudo touch /var/log/owid_fetch.log \ && sudo touch /var/log/foph_fetch.log \ && sudo touch /var/log/colombia_fetch.log \ - && sudo chown -R epigraphhub:epigraphhub /var/log/* - -COPY --chown=epigraphhub ./containers/airflow/airflow.cfg /opt/airflow/airflow.cfg -COPY --chown=epigraphhub ./containers/airflow/scripts/*.sh /opt/scripts/ -COPY --chown=epigraphhub ./containers/airflow/scripts/entrypoint.sh /opt/entrypoint.sh -COPY --chown=epigraphhub ./containers/airflow/scripts/webserver_config.py /opt/airflow/webserver_config.py + && sudo chown -R airflow /var/log/* ENTRYPOINT [ "/opt/entrypoint.sh" ] CMD /opt/scripts/startup.sh diff --git a/containers/airflow/README.md b/containers/airflow/README.md new file mode 100644 index 00000000..529bf218 --- /dev/null +++ b/containers/airflow/README.md @@ -0,0 +1,9 @@ +Building Airflow: +```sh +sugar build --group airflow +``` + +Starting containers: +```sh +sugar up --options -d --group airflow +``` diff --git a/containers/airflow/airflow.cfg b/containers/airflow/airflow.cfg deleted file mode 100644 index 5816b268..00000000 --- a/containers/airflow/airflow.cfg +++ /dev/null @@ -1,1234 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# This is the template for Airflow's default configuration. When Airflow is -# imported, it looks for a configuration file at $AIRFLOW_HOME/airflow.cfg. If -# it doesn't exist, Airflow uses this template to generate it by replacing -# variables in curly braces with their global values from configuration.py. - -# Users should not modify this file; they should customize the generated -# airflow.cfg instead. - - -# ----------------------- TEMPLATE BEGINS HERE ----------------------- - -[core] -# The folder where your airflow pipelines live, most likely a -# subfolder in a code repository. This path must be absolute. -dags_folder = /opt/airflow/dags - -# Hostname by providing a path to a callable, which will resolve the hostname. -# The format is "package.function". -# -# For example, default value "socket.getfqdn" means that result from getfqdn() of "socket" -# package will be used as hostname. -# -# No argument should be required in the function specified. -# If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address`` -hostname_callable = socket.getfqdn - -# Default timezone in case supplied date times are naive -# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) -default_timezone = utc - -# The executor class that airflow should use. Choices include -# ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``, -# ``KubernetesExecutor``, ``CeleryKubernetesExecutor`` or the -# full import path to the class when using a custom executor. -executor = LocalExecutor - -# This defines the maximum number of task instances that can run concurrently per scheduler in -# Airflow, regardless of the worker count. Generally this value, multiplied by the number of -# schedulers in your cluster, is the maximum number of task instances with the running -# state in the metadata database. -parallelism = 32 - -# The maximum number of task instances allowed to run concurrently in each DAG. To calculate -# the number of tasks that is running concurrently for a DAG, add up the number of running -# tasks for all DAG runs of the DAG. This is configurable at the DAG level with ``max_active_tasks``, -# which is defaulted as ``max_active_tasks_per_dag``. -# -# An example scenario when this would be useful is when you want to stop a new dag with an early -# start date from stealing all the executor slots in a cluster. -max_active_tasks_per_dag = 16 - -# Are DAGs paused by default at creation -dags_are_paused_at_creation = True - -# The maximum number of active DAG runs per DAG. The scheduler will not create more DAG runs -# if it reaches the limit. This is configurable at the DAG level with ``max_active_runs``, -# which is defaulted as ``max_active_runs_per_dag``. -max_active_runs_per_dag = 16 - -# Whether to load the DAG examples that ship with Airflow. It's good to -# get started, but you probably want to set this to ``False`` in a production -# environment -load_examples = False - -# Path to the folder containing Airflow plugins -plugins_folder = /opt/airflow/plugins - -# Should tasks be executed via forking of the parent process ("False", -# the speedier option) or by spawning a new python process ("True" slow, -# but means plugin changes picked up by tasks straight away) -execute_tasks_new_python_interpreter = False - -# Secret key to save connection passwords in the db -# fernet_key = - -# Whether to disable pickling dags -donot_pickle = True - -# How long before timing out a python file import -dagbag_import_timeout = 300.0 - -# Should a traceback be shown in the UI for dagbag import errors, -# instead of just the exception message -dagbag_import_error_tracebacks = True - -# If tracebacks are shown, how many entries from the traceback should be shown -dagbag_import_error_traceback_depth = 2 - -# How long before timing out a DagFileProcessor, which processes a dag file -dag_file_processor_timeout = 50 - -# The class to use for running task instances in a subprocess. -# Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class -# when using a custom task runner. -task_runner = StandardTaskRunner - -# If set, tasks without a ``run_as_user`` argument will be run with this user -# Can be used to de-elevate a sudo user running Airflow when executing tasks -default_impersonation = - -# What security module to use (for example kerberos) -security = - -# Turn unit test mode on (overwrites many configuration options with test -# values at runtime) -unit_test_mode = False - -# Whether to enable pickling for xcom (note that this is insecure and allows for -# RCE exploits). -enable_xcom_pickling = False - -# When a task is killed forcefully, this is the amount of time in seconds that -# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED -killed_task_cleanup_time = 60 - -# Whether to override params with dag_run.conf. If you pass some key-value pairs -# through ``airflow dags backfill -c`` or -# ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. -dag_run_conf_overrides_params = True - -# When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``. -dag_discovery_safe_mode = True - -# The pattern syntax used in the ".airflowignore" files in the DAG directories. Valid values are -# ``regexp`` or ``glob``. -dag_ignore_file_syntax = regexp - -# The number of retries each task is going to have by default. Can be overridden at dag or task level. -default_task_retries = 0 - -# The weighting method used for the effective total priority weight of the task -default_task_weight_rule = downstream - -# The default task execution_timeout value for the operators. Expected an integer value to -# be passed into timedelta as seconds. If not specified, then the value is considered as None, -# meaning that the operators are never timed out by default. -default_task_execution_timeout = - -# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. -min_serialized_dag_update_interval = 30 - -# If True, serialized DAGs are compressed before writing to DB. -# Note: this will disable the DAG dependencies view -compress_serialized_dags = False - -# Fetching serialized DAG can not be faster than a minimum interval to reduce database -# read rate. This config controls when your DAGs are updated in the Webserver -min_serialized_dag_fetch_interval = 10 - -# Maximum number of Rendered Task Instance Fields (Template Fields) per task to store -# in the Database. -# All the template_fields for each of Task Instance are stored in the Database. -# Keeping this number small may cause an error when you try to view ``Rendered`` tab in -# TaskInstance view for older tasks. -max_num_rendered_ti_fields_per_task = 30 - -# On each dagrun check against defined SLAs -check_slas = True - -# Path to custom XCom class that will be used to store and resolve operators results -# Example: xcom_backend = path.to.CustomXCom -xcom_backend = airflow.models.xcom.BaseXCom - -# By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``, -# if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module. -lazy_load_plugins = True - -# By default Airflow providers are lazily-discovered (discovery and imports happen only when required). -# Set it to False, if you want to discover providers whenever 'airflow' is invoked via cli or -# loaded from module. -lazy_discover_providers = True - -# Hide sensitive Variables or Connection extra json keys from UI and task logs when set to True -# -# (Connection passwords are always hidden in logs) -hide_sensitive_var_conn_fields = True - -# A comma-separated list of extra sensitive keywords to look for in variables names or connection's -# extra JSON. -sensitive_var_conn_names = - -# Task Slot counts for ``default_pool``. This setting would not have any effect in an existing -# deployment where the ``default_pool`` is already created. For existing deployments, users can -# change the number of slots using Webserver, API or the CLI -default_pool_task_slot_count = 128 - -# The maximum list/dict length an XCom can push to trigger task mapping. If the pushed list/dict has a -# length exceeding this value, the task pushing the XCom will be failed automatically to prevent the -# mapped tasks from clogging the scheduler. -max_map_length = 1024 - -[database] -# The SqlAlchemy connection string to the metadata database. -# SqlAlchemy supports many different database engines. -# More information here: -# http://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#database-uri -# sql_alchemy_conn = sqlite:////opt/airflow/airflow.db - -# Extra engine specific keyword args passed to SQLAlchemy's create_engine, as a JSON-encoded value -# Example: sql_alchemy_engine_args = {{"arg1": True}} -# sql_alchemy_engine_args = - -# The encoding for the databases -sql_engine_encoding = utf-8 - -# Collation for ``dag_id``, ``task_id``, ``key`` columns in case they have different encoding. -# By default this collation is the same as the database collation, however for ``mysql`` and ``mariadb`` -# the default is ``utf8mb3_bin`` so that the index sizes of our index keys will not exceed -# the maximum size of allowed index when collation is set to ``utf8mb4`` variant -# (see https://github.com/apache/airflow/pull/17603#issuecomment-901121618). -# sql_engine_collation_for_ids = - -# If SqlAlchemy should pool database connections. -sql_alchemy_pool_enabled = True - -# The SqlAlchemy pool size is the maximum number of database connections -# in the pool. 0 indicates no limit. -sql_alchemy_pool_size = 5 - -# The maximum overflow size of the pool. -# When the number of checked-out connections reaches the size set in pool_size, -# additional connections will be returned up to this limit. -# When those additional connections are returned to the pool, they are disconnected and discarded. -# It follows then that the total number of simultaneous connections the pool will allow -# is pool_size + max_overflow, -# and the total number of "sleeping" connections the pool will allow is pool_size. -# max_overflow can be set to ``-1`` to indicate no overflow limit; -# no limit will be placed on the total number of concurrent connections. Defaults to ``10``. -sql_alchemy_max_overflow = 10 - -# The SqlAlchemy pool recycle is the number of seconds a connection -# can be idle in the pool before it is invalidated. This config does -# not apply to sqlite. If the number of DB connections is ever exceeded, -# a lower config value will allow the system to recover faster. -sql_alchemy_pool_recycle = 1800 - -# Check connection at the start of each connection pool checkout. -# Typically, this is a simple statement like "SELECT 1". -# More information here: -# https://docs.sqlalchemy.org/en/14/core/pooling.html#disconnect-handling-pessimistic -sql_alchemy_pool_pre_ping = True - -# The schema to use for the metadata database. -# SqlAlchemy supports databases with the concept of multiple schemas. -sql_alchemy_schema = - -# Import path for connect args in SqlAlchemy. Defaults to an empty dict. -# This is useful when you want to configure db engine args that SqlAlchemy won't parse -# in connection string. -# See https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine.params.connect_args -# sql_alchemy_connect_args = - -# Whether to load the default connections that ship with Airflow. It's good to -# get started, but you probably want to set this to ``False`` in a production -# environment -load_default_connections = True - -# Number of times the code should be retried in case of DB Operational Errors. -# Not all transactions will be retried as it can cause undesired state. -# Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. -max_db_retries = 3 - -[logging] -# The folder where airflow should store its log files. -# This path must be absolute. -# There are a few existing configurations that assume this is set to the default. -# If you choose to override this you may need to update the dag_processor_manager_log_location and -# dag_processor_manager_log_location settings as well. -base_log_folder = /opt/airflow/logs - -# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. -# Set this to True if you want to enable remote logging. -remote_logging = False - -# Users must supply an Airflow connection id that provides access to the storage -# location. Depending on your remote logging service, this may only be used for -# reading logs, not writing them. -remote_log_conn_id = - -# Path to Google Credential JSON file. If omitted, authorization based on `the Application Default -# Credentials -# `__ will -# be used. -google_key_path = - -# Storage bucket URL for remote logging -# S3 buckets should start with "s3://" -# Cloudwatch log groups should start with "cloudwatch://" -# GCS buckets should start with "gs://" -# WASB buckets should start with "wasb" just to help Airflow select correct handler -# Stackdriver logs should start with "stackdriver://" -remote_base_log_folder = - -# Use server-side encryption for logs stored in S3 -encrypt_s3_logs = False - -# Logging level. -# -# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. -logging_level = DEBUG - -# Logging level for celery. If not set, it uses the value of logging_level -# -# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. -celery_logging_level = - -# Logging level for Flask-appbuilder UI. -# -# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. -fab_logging_level = WARNING - -# Logging class -# Specify the class that will specify the logging configuration -# This class has to be on the python classpath -# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG -logging_config_class = - -# Flag to enable/disable Colored logs in Console -# Colour the logs when the controlling terminal is a TTY. -colored_console_log = True - -# Log format for when Colored logs is enabled -colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s -colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter - -# Format of Log line -log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s -simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s - -# Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter -# Example: task_log_prefix_template = {{ti.dag_id}}-{{ti.task_id}}-{{execution_date}}-{{try_number}} -task_log_prefix_template = - -# Formatting for how airflow generates file names/paths for each task run. -# log_filename_template = dag_id={{{{ ti.dag_id }}}}/run_id={{{{ ti.run_id }}}}/task_id={{{{ ti.task_id }}}}/{{%% if ti.map_index >= 0 %%}}map_index={{{{ ti.map_index }}}}/{{%% endif %%}}attempt={{{{ try_number }}}}.log - -# Formatting for how airflow generates file names for log -# log_processor_filename_template = {{{{ filename }}}}.log - -# Full path of dag_processor_manager logfile. -dag_processor_manager_log_location = /opt/airflow/logs/dag_processor_manager/dag_processor_manager.log - -# Name of handler to read task instance logs. -# Defaults to use ``task`` handler. -task_log_reader = task - -# A comma\-separated list of third-party logger names that will be configured to print messages to -# consoles\. -# Example: extra_logger_names = connexion,sqlalchemy -extra_logger_names = - -# When you start an airflow worker, airflow starts a tiny web server -# subprocess to serve the workers local log files to the airflow main -# web server, who then builds pages and sends them to users. This defines -# the port on which the logs are served. It needs to be unused, and open -# visible from the main web server to connect into the workers. -worker_log_server_port = 8793 - -[metrics] - -# StatsD (https://github.com/etsy/statsd) integration settings. -# Enables sending metrics to StatsD. -statsd_on = False -statsd_host = localhost -statsd_port = 8125 -statsd_prefix = airflow - -# If you want to avoid sending all the available metrics to StatsD, -# you can configure an allow list of prefixes (comma separated) to send only the metrics that -# start with the elements of the list (e.g: "scheduler,executor,dagrun") -statsd_allow_list = - -# A function that validate the StatsD stat name, apply changes to the stat name if necessary and return -# the transformed stat name. -# -# The function should have the following signature: -# def func_name(stat_name: str) -> str: -stat_name_handler = - -# To enable datadog integration to send airflow metrics. -statsd_datadog_enabled = False - -# List of datadog tags attached to all metrics(e.g: key1:value1,key2:value2) -statsd_datadog_tags = - -# If you want to utilise your own custom StatsD client set the relevant -# module path below. -# Note: The module path must exist on your PYTHONPATH for Airflow to pick it up -# statsd_custom_client_path = - -[secrets] -# Full class name of secrets backend to enable (will precede env vars and metastore in search path) -# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend -backend = - -# The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. -# See documentation for the secrets backend you are using. JSON is expected. -# Example for AWS Systems Manager ParameterStore: -# ``{{"connections_prefix": "/airflow/connections", "profile_name": "default"}}`` -backend_kwargs = - -[cli] -# In what way should the cli access the API. The LocalClient will use the -# database directly, while the json_client will use the api running on the -# webserver -api_client = airflow.api.client.local_client - -# If you set web_server_url_prefix, do NOT forget to append it here, ex: -# ``endpoint_url = http://localhost:8080/myroot`` -# So api will look like: ``http://localhost:8080/myroot/api/experimental/...`` -endpoint_url = http://localhost:8080 - -[debug] -# Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first -# failed task. Helpful for debugging purposes. -fail_fast = False - -[api] -# Enables the deprecated experimental API. Please note that these APIs do not have access control. -# The authenticated user has full access. -# -# .. warning:: -# -# This `Experimental REST API `__ is -# deprecated since version 2.0. Please consider using -# `the Stable REST API `__. -# For more information on migration, see -# `RELEASE_NOTES.rst `_ -enable_experimental_api = False - -# Comma separated list of auth backends to authenticate users of the API. See -# https://airflow.apache.org/docs/apache-airflow/stable/security/api.html for possible values. -# ("airflow.api.auth.backend.default" allows all requests for historic reasons) -auth_backends = airflow.api.auth.backend.session - -# Used to set the maximum page limit for API requests -maximum_page_limit = 100 - -# Used to set the default page limit when limit is zero. A default limit -# of 100 is set on OpenApi spec. However, this particular default limit -# only work when limit is set equal to zero(0) from API requests. -# If no limit is supplied, the OpenApi spec default is used. -fallback_page_limit = 100 - -# The intended audience for JWT token credentials used for authorization. This value must match on the client and server sides. If empty, audience will not be tested. -# Example: google_oauth2_audience = project-id-random-value.apps.googleusercontent.com -google_oauth2_audience = - -# Path to Google Cloud Service Account key file (JSON). If omitted, authorization based on -# `the Application Default Credentials -# `__ will -# be used. -# Example: google_key_path = /files/service-account-json -google_key_path = - -# Used in response to a preflight request to indicate which HTTP -# headers can be used when making the actual request. This header is -# the server side response to the browser's -# Access-Control-Request-Headers header. -access_control_allow_headers = - -# Specifies the method or methods allowed when accessing the resource. -access_control_allow_methods = - -# Indicates whether the response can be shared with requesting code from the given origins. -# Separate URLs with space. -access_control_allow_origins = - -[lineage] -# what lineage backend to use -backend = - -[atlas] -sasl_enabled = False -host = -port = 21000 -username = -password = - -[operators] -# The default owner assigned to each new operator, unless -# provided explicitly or passed via ``default_args`` -default_owner = airflow -default_cpus = 1 -default_ram = 512 -default_disk = 512 -default_gpus = 0 - -# Default queue that tasks get assigned to and that worker listen on. -default_queue = default - -# Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator. -# If set to False, an exception will be thrown, otherwise only the console message will be displayed. -allow_illegal_arguments = False - -[hive] -# Default mapreduce queue for HiveOperator tasks -default_hive_mapred_queue = - -# Template for mapred_job_name in HiveOperator, supports the following named parameters -# hostname, dag_id, task_id, execution_date -# mapred_job_name_template = - -[webserver] -# The base url of your website as airflow cannot guess what domain or -# cname you are using. This is used in automated emails that -# airflow sends to point links to the right web server -base_url = http://localhost:8080 - -# Default timezone to display all dates in the UI, can be UTC, system, or -# any IANA timezone string (e.g. Europe/Amsterdam). If left empty the -# default value of core/default_timezone will be used -# Example: default_ui_timezone = America/New_York -default_ui_timezone = UTC - -# The ip specified when starting the web server -web_server_host = 0.0.0.0 - -# The port on which to run the web server -web_server_port = 8080 - -# Paths to the SSL certificate and key for the web server. When both are -# provided SSL will be enabled. This does not change the web server port. -web_server_ssl_cert = - -# Paths to the SSL certificate and key for the web server. When both are -# provided SSL will be enabled. This does not change the web server port. -web_server_ssl_key = - -# The type of backend used to store web session data, can be 'database' or 'securecookie' -# Example: session_backend = securecookie -session_backend = database - -# Number of seconds the webserver waits before killing gunicorn master that doesn't respond -web_server_master_timeout = 120 - -# Number of seconds the gunicorn webserver waits before timing out on a worker -web_server_worker_timeout = 120 - -# Number of workers to refresh at a time. When set to 0, worker refresh is -# disabled. When nonzero, airflow periodically refreshes webserver workers by -# bringing up new ones and killing old ones. -worker_refresh_batch_size = 1 - -# Number of seconds to wait before refreshing a batch of workers. -worker_refresh_interval = 6000 - -# If set to True, Airflow will track files in plugins_folder directory. When it detects changes, -# then reload the gunicorn. -reload_on_plugin_change = False - -# Secret key used to run your flask app. It should be as random as possible. However, when running -# more than 1 instances of webserver, make sure all of them use the same ``secret_key`` otherwise -# one of them will error with "CSRF session token is missing". -# The webserver key is also used to authorize requests to Celery workers when logs are retrieved. -# The token generated using the secret key has a short expiry time though - make sure that time on -# ALL the machines that you run airflow components on is synchronized (for example using ntpd) -# otherwise you might get "forbidden" errors when the logs are accessed. -# secret_key = - -# Number of workers to run the Gunicorn web server -workers = 4 - -# The worker class gunicorn should use. Choices include -# sync (default), eventlet, gevent -worker_class = sync - -# Log files for the gunicorn webserver. '-' means log to stderr. -access_logfile = - - -# Log files for the gunicorn webserver. '-' means log to stderr. -error_logfile = - - -# Access log format for gunicorn webserver. -# default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s" -# documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format -access_logformat = - -# Expose the configuration file in the web server -expose_config = False - -# Expose hostname in the web server -expose_hostname = True - -# Expose stacktrace in the web server -expose_stacktrace = True - -# Default DAG view. Valid values are: ``grid``, ``graph``, ``duration``, ``gantt``, ``landing_times`` -dag_default_view = grid - -# Default DAG orientation. Valid values are: -# ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT`` (Bottom->Top) -dag_orientation = LR - -# The amount of time (in secs) webserver will wait for initial handshake -# while fetching logs from other worker machine -log_fetch_timeout_sec = 5 - -# Time interval (in secs) to wait before next log fetching. -log_fetch_delay_sec = 2 - -# Distance away from page bottom to enable auto tailing. -log_auto_tailing_offset = 30 - -# Animation speed for auto tailing log display. -log_animation_speed = 1000 - -# By default, the webserver shows paused DAGs. Flip this to hide paused -# DAGs by default -hide_paused_dags_by_default = False - -# Consistent page size across all listing views in the UI -page_size = 100 - -# Define the color of navigation bar -navbar_color = #fff - -# Default dagrun to show in UI -default_dag_run_display_number = 25 - -# Enable werkzeug ``ProxyFix`` middleware for reverse proxy -enable_proxy_fix = False - -# Number of values to trust for ``X-Forwarded-For``. -# More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/ -proxy_fix_x_for = 1 - -# Number of values to trust for ``X-Forwarded-Proto`` -proxy_fix_x_proto = 1 - -# Number of values to trust for ``X-Forwarded-Host`` -proxy_fix_x_host = 1 - -# Number of values to trust for ``X-Forwarded-Port`` -proxy_fix_x_port = 1 - -# Number of values to trust for ``X-Forwarded-Prefix`` -proxy_fix_x_prefix = 1 - -# Set secure flag on session cookie -cookie_secure = False - -# Set samesite policy on session cookie -cookie_samesite = Lax - -# Default setting for wrap toggle on DAG code and TI log views. -default_wrap = False - -# Allow the UI to be rendered in a frame -x_frame_enabled = True - -# Send anonymous user activity to your analytics tool -# choose from google_analytics, segment, or metarouter -# analytics_tool = - -# Unique ID of your account in the analytics tool -# analytics_id = - -# 'Recent Tasks' stats will show for old DagRuns if set -show_recent_stats_for_completed_runs = True - -# Update FAB permissions and sync security manager roles -# on webserver startup -update_fab_perms = True - -# The UI cookie lifetime in minutes. User will be logged out from UI after -# ``session_lifetime_minutes`` of non-activity -session_lifetime_minutes = 43200 - -# Sets a custom page title for the DAGs overview page and site title for all pages -# instance_name = - -# Whether the custom page title for the DAGs overview page contains any Markup language -instance_name_has_markup = False - -# How frequently, in seconds, the DAG data will auto-refresh in graph or grid view -# when auto-refresh is turned on -auto_refresh_interval = 3 - -# Boolean for displaying warning for publicly viewable deployment -warn_deployment_exposure = True - -# Comma separated string of view events to exclude from dag audit view. -# All other events will be added minus the ones passed here. -# The audit logs in the db will not be affected by this parameter. -audit_view_excluded_events = gantt,landing_times,tries,duration,calendar,graph,grid,tree,tree_data - -# Comma separated string of view events to include in dag audit view. -# If passed, only these events will populate the dag audit view. -# The audit logs in the db will not be affected by this parameter. -# Example: audit_view_included_events = dagrun_cleared,failed -# audit_view_included_events = - -[email] - -# Configuration email backend and whether to -# send email alerts on retry or failure -# Email backend to use -email_backend = airflow.utils.email.send_email_smtp - -# Email connection to use -email_conn_id = smtp_default - -# Whether email alerts should be sent when a task is retried -default_email_on_retry = True - -# Whether email alerts should be sent when a task failed -default_email_on_failure = True - -# File that will be used as the template for Email subject (which will be rendered using Jinja2). -# If not set, Airflow uses a base template. -# Example: subject_template = /path/to/my_subject_template_file -# subject_template = - -# File that will be used as the template for Email content (which will be rendered using Jinja2). -# If not set, Airflow uses a base template. -# Example: html_content_template = /path/to/my_html_content_template_file -# html_content_template = - -# Email address that will be used as sender address. -# It can either be raw email or the complete address in a format ``Sender Name `` -# Example: from_email = Airflow -# from_email = - -[smtp] - -# If you want airflow to send emails on retries, failure, and you want to use -# the airflow.utils.email.send_email_smtp function, you have to configure an -# smtp server here -smtp_host = localhost -smtp_starttls = True -smtp_ssl = False -# Example: smtp_user = airflow -# smtp_user = -# Example: smtp_password = airflow -# smtp_password = -smtp_port = 25 -smtp_mail_from = airflow@example.com -smtp_timeout = 30 -smtp_retry_limit = 5 - -[sentry] - -# Sentry (https://docs.sentry.io) integration. Here you can supply -# additional configuration options based on the Python platform. See: -# https://docs.sentry.io/error-reporting/configuration/?platform=python. -# Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``, -# ``ignore_errors``, ``before_breadcrumb``, ``transport``. -# Enable error reporting to Sentry -sentry_on = false -sentry_dsn = - -# Dotted path to a before_send function that the sentry SDK should be configured to use. -# before_send = - -[local_kubernetes_executor] - -# This section only applies if you are using the ``LocalKubernetesExecutor`` in -# ``[core]`` section above -# Define when to send a task to ``KubernetesExecutor`` when using ``LocalKubernetesExecutor``. -# When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``), -# the task is executed via ``KubernetesExecutor``, -# otherwise via ``LocalExecutor`` -kubernetes_queue = kubernetes - -[celery_kubernetes_executor] - -# This section only applies if you are using the ``CeleryKubernetesExecutor`` in -# ``[core]`` section above -# Define when to send a task to ``KubernetesExecutor`` when using ``CeleryKubernetesExecutor``. -# When the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``), -# the task is executed via ``KubernetesExecutor``, -# otherwise via ``CeleryExecutor`` -kubernetes_queue = kubernetes - -[celery] - -# This section only applies if you are using the CeleryExecutor in -# ``[core]`` section above -# The app name that will be used by celery -celery_app_name = airflow.executors.celery_executor - -# The concurrency that will be used when starting workers with the -# ``airflow celery worker`` command. This defines the number of task instances that -# a worker will take, so size up your workers based on the resources on -# your worker box and the nature of your tasks -worker_concurrency = 16 - -# The maximum and minimum concurrency that will be used when starting workers with the -# ``airflow celery worker`` command (always keep minimum processes, but grow -# to maximum if necessary). Note the value should be max_concurrency,min_concurrency -# Pick these numbers based on resources on worker box and the nature of the task. -# If autoscale option is available, worker_concurrency will be ignored. -# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale -# Example: worker_autoscale = 16,12 -# worker_autoscale = - -# Used to increase the number of tasks that a worker prefetches which can improve performance. -# The number of processes multiplied by worker_prefetch_multiplier is the number of tasks -# that are prefetched by a worker. A value greater than 1 can result in tasks being unnecessarily -# blocked if there are multiple workers and one worker prefetches tasks that sit behind long -# running tasks while another worker has unutilized processes that are unable to process the already -# claimed blocked tasks. -# https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits -worker_prefetch_multiplier = 1 - -# Specify if remote control of the workers is enabled. -# When using Amazon SQS as the broker, Celery creates lots of ``.*reply-celery-pidbox`` queues. You can -# prevent this by setting this to false. However, with this disabled Flower won't work. -worker_enable_remote_control = true - -# Umask that will be used when starting workers with the ``airflow celery worker`` -# in daemon mode. This control the file-creation mode mask which determines the initial -# value of file permission bits for newly created files. -worker_umask = 0o077 - -# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally -# a sqlalchemy database. Refer to the Celery documentation for more information. -broker_url = redis://redis:6379/0 - -# The Celery result_backend. When a job finishes, it needs to update the -# metadata of the job. Therefore it will post a message on a message bus, -# or insert it into a database (depending of the backend) -# This status is used by the scheduler to update the state of the task -# The use of a database is highly recommended -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings -result_backend = db+postgresql://postgres:airflow@postgres/airflow - -# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start -# it ``airflow celery flower``. This defines the IP that Celery Flower runs on -flower_host = 0.0.0.0 - -# The root URL for Flower -# Example: flower_url_prefix = /flower -flower_url_prefix = - -# This defines the port that Celery Flower runs on -flower_port = 5555 - -# Securing Flower with Basic Authentication -# Accepts user:password pairs separated by a comma -# Example: flower_basic_auth = user1:password1,user2:password2 -flower_basic_auth = - -# How many processes CeleryExecutor uses to sync task state. -# 0 means to use max(1, number of cores - 1) processes. -sync_parallelism = 0 - -# Import path for celery configuration options -celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG -ssl_active = False -ssl_key = -ssl_cert = -ssl_cacert = - -# Celery Pool implementation. -# Choices include: ``prefork`` (default), ``eventlet``, ``gevent`` or ``solo``. -# See: -# https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency -# https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html -pool = prefork - -# The number of seconds to wait before timing out ``send_task_to_executor`` or -# ``fetch_celery_task_state`` operations. -operation_timeout = 5.0 - -# Celery task will report its status as 'started' when the task is executed by a worker. -# This is used in Airflow to keep track of the running tasks and if a Scheduler is restarted -# or run in HA mode, it can adopt the orphan tasks launched by previous SchedulerJob. -task_track_started = True - -# Time in seconds after which adopted tasks which are queued in celery are assumed to be stalled, -# and are automatically rescheduled. This setting does the same thing as ``stalled_task_timeout`` but -# applies specifically to adopted tasks only. When set to 0, the ``stalled_task_timeout`` setting -# also applies to adopted tasks. -task_adoption_timeout = 600 - -# Time in seconds after which tasks queued in celery are assumed to be stalled, and are automatically -# rescheduled. Adopted tasks will instead use the ``task_adoption_timeout`` setting if specified. -# When set to 0, automatic clearing of stalled tasks is disabled. -stalled_task_timeout = 0 - -# The Maximum number of retries for publishing task messages to the broker when failing -# due to ``AirflowTaskTimeout`` error before giving up and marking Task as failed. -task_publish_max_retries = 3 - -# Worker initialisation check to validate Metadata Database connection -worker_precheck = False - -[celery_broker_transport_options] - -# This section is for specifying options which can be passed to the -# underlying celery broker transport. See: -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options -# The visibility timeout defines the number of seconds to wait for the worker -# to acknowledge the task before the message is redelivered to another worker. -# Make sure to increase the visibility timeout to match the time of the longest -# ETA you're planning to use. -# visibility_timeout is only supported for Redis and SQS celery brokers. -# See: -# http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options -# Example: visibility_timeout = 21600 -# visibility_timeout = - -[dask] - -# This section only applies if you are using the DaskExecutor in -# [core] section above -# The IP address and port of the Dask cluster's scheduler. -cluster_address = 127.0.0.1:8786 - -# TLS/ SSL settings to access a secured Dask scheduler. -tls_ca = -tls_cert = -tls_key = - -[scheduler] -# Task instances listen for external kill signal (when you clear tasks -# from the CLI or the UI), this defines the frequency at which they should -# listen (in seconds). -job_heartbeat_sec = 5 - -# The scheduler constantly tries to trigger new tasks (look at the -# scheduler section in the docs for more information). This defines -# how often the scheduler should run (in seconds). -scheduler_heartbeat_sec = 5 - -# The number of times to try to schedule each DAG file -# -1 indicates unlimited number -num_runs = -1 - -# Controls how long the scheduler will sleep between loops, but if there was nothing to do -# in the loop. i.e. if it scheduled something then it will start the next loop -# iteration straight away. -scheduler_idle_sleep_time = 1 - -# Number of seconds after which a DAG file is parsed. The DAG file is parsed every -# ``min_file_process_interval`` number of seconds. Updates to DAGs are reflected after -# this interval. Keeping this number low will increase CPU usage. -min_file_process_interval = 30 - -# How often (in seconds) to check for stale DAGs (DAGs which are no longer present in -# the expected files) which should be deactivated. -parsing_cleanup_interval = 60 - -# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. -dag_dir_list_interval = 300 - -# How often should stats be printed to the logs. Setting to 0 will disable printing stats -print_stats_interval = 30 - -# How often (in seconds) should pool usage stats be sent to StatsD (if statsd_on is enabled) -pool_metrics_interval = 5.0 - -# If the last scheduler heartbeat happened more than scheduler_health_check_threshold -# ago (in seconds), scheduler is considered unhealthy. -# This is used by the health check in the "/health" endpoint -scheduler_health_check_threshold = 30 - -# How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs -orphaned_tasks_check_interval = 300.0 -child_process_log_directory = /opt/airflow/logs/scheduler - -# Local task jobs periodically heartbeat to the DB. If the job has -# not heartbeat in this many seconds, the scheduler will mark the -# associated task instance as failed and will re-schedule the task. -scheduler_zombie_task_threshold = 300 - -# How often (in seconds) should the scheduler check for zombie tasks. -zombie_detection_interval = 10.0 - -# Turn off scheduler catchup by setting this to ``False``. -# Default behavior is unchanged and -# Command Line Backfills still work, but the scheduler -# will not do scheduler catchup if this is ``False``, -# however it can be set on a per DAG basis in the -# DAG definition (catchup) -catchup_by_default = True - -# Setting this to True will make first task instance of a task -# ignore depends_on_past setting. A task instance will be considered -# as the first task instance of a task when there is no task instance -# in the DB with an execution_date earlier than it., i.e. no manual marking -# success will be needed for a newly added task to be scheduled. -ignore_first_depends_on_past_by_default = True - -# This changes the batch size of queries in the scheduling main loop. -# If this is too high, SQL query performance may be impacted by -# complexity of query predicate, and/or excessive locking. -# Additionally, you may hit the maximum allowable query length for your db. -# Set this to 0 for no limit (not advised) -max_tis_per_query = 512 - -# Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. -# If this is set to False then you should not run more than a single -# scheduler at once -use_row_level_locking = True - -# Max number of DAGs to create DagRuns for per scheduler loop. -max_dagruns_to_create_per_loop = 10 - -# How many DagRuns should a scheduler examine (and lock) when scheduling -# and queuing tasks. -max_dagruns_per_loop_to_schedule = 20 - -# Should the Task supervisor process perform a "mini scheduler" to attempt to schedule more tasks of the -# same DAG. Leaving this on will mean tasks in the same DAG execute quicker, but might starve out other -# dags in some circumstances -schedule_after_task_execution = True - -# The scheduler can run multiple processes in parallel to parse dags. -# This defines how many processes will run. -parsing_processes = 2 - -# One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``. -# The scheduler will list and sort the dag files to decide the parsing order. -# -# * ``modified_time``: Sort by modified time of the files. This is useful on large scale to parse the -# recently modified DAGs first. -# * ``random_seeded_by_host``: Sort randomly across multiple Schedulers but with same order on the -# same host. This is useful when running with Scheduler in HA mode where each scheduler can -# parse different DAG files. -# * ``alphabetical``: Sort by filename -file_parsing_sort_mode = modified_time - -# Whether the dag processor is running as a standalone process or it is a subprocess of a scheduler -# job. -standalone_dag_processor = False - -# Only applicable if `[scheduler]standalone_dag_processor` is true and callbacks are stored -# in database. Contains maximum number of callbacks that are fetched during a single loop. -max_callbacks_per_loop = 20 - -# Turn off scheduler use of cron intervals by setting this to False. -# DAGs submitted manually in the web UI or with trigger_dag will still run. -use_job_schedule = True - -# Allow externally triggered DagRuns for Execution Dates in the future -# Only has effect if schedule_interval is set to None in DAG -allow_trigger_in_future = False - -# DAG dependency detector class to use -dependency_detector = airflow.serialization.serialized_objects.DependencyDetector - -# How often to check for expired trigger requests that have not run yet. -trigger_timeout_check_interval = 15 - -[triggerer] -# How many triggers a single Triggerer will run at once, by default. -default_capacity = 1000 - -# [kerberos] -# ccache = /tmp/airflow_krb5_ccache -# -# # gets augmented with fqdn -# principal = airflow -# reinit_frequency = 3600 -# kinit_path = kinit -# keytab = airflow.keytab -# -# # Allow to disable ticket forwardability. -# forwardable = True -# -# # Allow to remove source IP from token, useful when using token behind NATted Docker host. -# include_ip = True - -[github_enterprise] -api_rev = v3 - -[elasticsearch] -# Elasticsearch host -host = - -# Format of the log_id, which is used to query for a given tasks logs -log_id_template = {{dag_id}}-{{task_id}}-{{run_id}}-{{map_index}}-{{try_number}} - -# Used to mark the end of a log stream for a task -end_of_log_mark = end_of_log - -# Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id -# Code will construct log_id using the log_id template from the argument above. -# NOTE: scheme will default to https if one is not provided -# Example: frontend = http://localhost:5601/app/kibana#/discover?_a=(columns:!(message),query:(language:kuery,query:'log_id: "{{log_id}}"'),sort:!(log.offset,asc)) -frontend = - -# Write the task logs to the stdout of the worker, rather than the default files -write_stdout = False - -# Instead of the default log formatter, write the log lines as JSON -json_format = False - -# Log fields to also attach to the json output, if enabled -json_fields = asctime, filename, lineno, levelname, message - -# The field where host name is stored (normally either `host` or `host.name`) -host_field = host - -# The field where offset is stored (normally either `offset` or `log.offset`) -offset_field = offset - -[elasticsearch_configs] -use_ssl = False -verify_certs = True - -[kubernetes] -# Path to the YAML pod file that forms the basis for KubernetesExecutor workers. -pod_template_file = - -# The repository of the Kubernetes Image for the Worker to Run -worker_container_repository = - -# The tag of the Kubernetes Image for the Worker to Run -worker_container_tag = - -# The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` -namespace = default - -# If True, all worker pods will be deleted upon termination -delete_worker_pods = True - -# If False (and delete_worker_pods is True), -# failed worker pods will not be deleted so users can investigate them. -# This only prevents removal of worker pods where the worker itself failed, -# not when the task it ran failed. -delete_worker_pods_on_failure = False - -# Number of Kubernetes Worker Pod creation calls per scheduler loop. -# Note that the current default of "1" will only launch a single pod -# per-heartbeat. It is HIGHLY recommended that users increase this -# number to match the tolerance of their kubernetes cluster for -# better performance. -worker_pods_creation_batch_size = 1 - -# Allows users to launch pods in multiple namespaces. -# Will require creating a cluster-role for the scheduler -multi_namespace_mode = False - -# Use the service account kubernetes gives to pods to connect to kubernetes cluster. -# It's intended for clients that expect to be running inside a pod running on kubernetes. -# It will raise an exception if called from a process not running in a kubernetes environment. -in_cluster = True - -# When running with in_cluster=False change the default cluster_context or config_file -# options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. -# cluster_context = - -# Path to the kubernetes configfile to be used when ``in_cluster`` is set to False -# config_file = - -# Keyword parameters to pass while calling a kubernetes client core_v1_api methods -# from Kubernetes Executor provided as a single line formatted JSON dictionary string. -# List of supported params are similar for all core_v1_apis, hence a single config -# variable for all apis. See: -# https://raw.githubusercontent.com/kubernetes-client/python/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/api/core_v1_api.py -kube_client_request_args = - -# Optional keyword arguments to pass to the ``delete_namespaced_pod`` kubernetes client -# ``core_v1_api`` method when using the Kubernetes Executor. -# This should be an object and can contain any of the options listed in the ``v1DeleteOptions`` -# class defined here: -# https://github.com/kubernetes-client/python/blob/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/models/v1_delete_options.py#L19 -# Example: delete_option_kwargs = {{"grace_period_seconds": 10}} -delete_option_kwargs = - -# Enables TCP keepalive mechanism. This prevents Kubernetes API requests to hang indefinitely -# when idle connection is time-outed on services like cloud load balancers or firewalls. -enable_tcp_keepalive = True - -# When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has -# been idle for `tcp_keep_idle` seconds. -tcp_keep_idle = 120 - -# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond -# to a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds. -tcp_keep_intvl = 30 - -# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond -# to a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before -# a connection is considered to be broken. -tcp_keep_cnt = 6 - -# Set this to false to skip verifying SSL certificate of Kubernetes python client. -verify_ssl = True - -# How long in seconds a worker can be in Pending before it is considered a failure -worker_pods_pending_timeout = 300 - -# How often in seconds to check if Pending workers have exceeded their timeouts -worker_pods_pending_timeout_check_interval = 120 - -# How often in seconds to check for task instances stuck in "queued" status without a pod -worker_pods_queued_check_interval = 60 - -# How many pending pods to check for timeout violations in each check interval. -# You may want this higher if you have a very large cluster and/or use ``multi_namespace_mode``. -worker_pods_pending_timeout_batch_size = 100 - -[sensors] -# Sensor default timeout, 7 days by default (7 * 24 * 60 * 60). -default_timeout = 604800 - -[smart_sensor] -# When `use_smart_sensor` is True, Airflow redirects multiple qualified sensor tasks to -# smart sensor task. -use_smart_sensor = False - -# `shard_code_upper_limit` is the upper limit of `shard_code` value. The `shard_code` is generated -# by `hashcode % shard_code_upper_limit`. -shard_code_upper_limit = 10000 - -# The number of running smart sensor processes for each service. -shards = 5 - -# comma separated sensor classes support in smart_sensor. -sensors_enabled = NamedHivePartitionSensor diff --git a/containers/airflow/config/airflow.cfg b/containers/airflow/config/airflow.cfg new file mode 100644 index 00000000..c70b26e4 --- /dev/null +++ b/containers/airflow/config/airflow.cfg @@ -0,0 +1,2080 @@ +[core] +# The folder where your airflow pipelines live, most likely a +# subfolder in a code repository. This path must be absolute. +# +# Variable: AIRFLOW__CORE__DAGS_FOLDER +# +dags_folder = $AIRFLOW_HOME/dags + +# Hostname by providing a path to a callable, which will resolve the hostname. +# The format is "package.function". +# +# For example, default value "airflow.utils.net.getfqdn" means that result from patched +# version of socket.getfqdn() - see https://github.com/python/cpython/issues/49254. +# +# No argument should be required in the function specified. +# If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address`` +# +# Variable: AIRFLOW__CORE__HOSTNAME_CALLABLE +# +# hostname_callable = airflow.utils.net.getfqdn + +# A callable to check if a python file has airflow dags defined or not +# with argument as: `(file_path: str, zip_file: zipfile.ZipFile | None = None)` +# return True if it has dags otherwise False +# If this is not provided, Airflow uses its own heuristic rules. +# +# Variable: AIRFLOW__CORE__MIGHT_CONTAIN_DAG_CALLABLE +# +# might_contain_dag_callable = airflow.utils.file.might_contain_dag_via_default_heuristic + +# Default timezone in case supplied date times are naive +# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) +# +# Variable: AIRFLOW__CORE__DEFAULT_TIMEZONE +# +default_timezone = utc + +# The executor class that airflow should use. Choices include +# ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``, +# ``KubernetesExecutor``, ``CeleryKubernetesExecutor`` or the +# full import path to the class when using a custom executor. +# +# Variable: AIRFLOW__CORE__EXECUTOR +# +executor = CeleryExecutor + +# The auth manager class that airflow should use. Full import path to the auth manager class. +# +# Variable: AIRFLOW__CORE__AUTH_MANAGER +# +# auth_manager = airflow.auth.managers.fab.fab_auth_manager.FabAuthManager + +# This defines the maximum number of task instances that can run concurrently per scheduler in +# Airflow, regardless of the worker count. Generally this value, multiplied by the number of +# schedulers in your cluster, is the maximum number of task instances with the running +# state in the metadata database. +# +# Variable: AIRFLOW__CORE__PARALLELISM +# +parallelism = 32 + +# The maximum number of task instances allowed to run concurrently in each DAG. To calculate +# the number of tasks that is running concurrently for a DAG, add up the number of running +# tasks for all DAG runs of the DAG. This is configurable at the DAG level with ``max_active_tasks``, +# which is defaulted as ``max_active_tasks_per_dag``. +# +# An example scenario when this would be useful is when you want to stop a new dag with an early +# start date from stealing all the executor slots in a cluster. +# +# Variable: AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG +# +max_active_tasks_per_dag = 16 + +# Are DAGs paused by default at creation +# +# Variable: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION +# +dags_are_paused_at_creation = True + +# The maximum number of active DAG runs per DAG. The scheduler will not create more DAG runs +# if it reaches the limit. This is configurable at the DAG level with ``max_active_runs``, +# which is defaulted as ``max_active_runs_per_dag``. +# +# Variable: AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG +# +max_active_runs_per_dag = 16 + +# The name of the method used in order to start Python processes via the multiprocessing module. +# This corresponds directly with the options available in the Python docs: +# https://docs.python.org/3/library/multiprocessing.html#multiprocessing.set_start_method. +# Must be one of the values returned by: +# https://docs.python.org/3/library/multiprocessing.html#multiprocessing.get_all_start_methods. +# +# Example: mp_start_method = fork +# +# Variable: AIRFLOW__CORE__MP_START_METHOD +# +# mp_start_method = + +# Whether to load the DAG examples that ship with Airflow. It's good to +# get started, but you probably want to set this to ``False`` in a production +# environment +# +# Variable: AIRFLOW__CORE__LOAD_EXAMPLES +# +load_examples = False + +# Path to the folder containing Airflow plugins +# +# Variable: AIRFLOW__CORE__PLUGINS_FOLDER +# +plugins_folder = $AIRFLOW_HOME/plugins + +# Should tasks be executed via forking of the parent process ("False", +# the speedier option) or by spawning a new python process ("True" slow, +# but means plugin changes picked up by tasks straight away) +# +# Variable: AIRFLOW__CORE__EXECUTE_TASKS_NEW_PYTHON_INTERPRETER +# +# execute_tasks_new_python_interpreter = False + +# Secret key to save connection passwords in the db +# +# Variable: AIRFLOW__CORE__FERNET_KEY +# +fernet_key = $AIRFLOW__CORE__FERNET_KEY + +# Whether to disable pickling dags +# +# Variable: AIRFLOW__CORE__DONOT_PICKLE +# +# donot_pickle = True + +# How long before timing out a python file import +# +# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT +# +dagbag_import_timeout = 30.0 + +# Should a traceback be shown in the UI for dagbag import errors, +# instead of just the exception message +# +# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_ERROR_TRACEBACKS +# +dagbag_import_error_tracebacks = True + +# If tracebacks are shown, how many entries from the traceback should be shown +# +# Variable: AIRFLOW__CORE__DAGBAG_IMPORT_ERROR_TRACEBACK_DEPTH +# +dagbag_import_error_traceback_depth = 2 + +# How long before timing out a DagFileProcessor, which processes a dag file +# +# Variable: AIRFLOW__CORE__DAG_FILE_PROCESSOR_TIMEOUT +# +dag_file_processor_timeout = 50 + +# The class to use for running task instances in a subprocess. +# Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class +# when using a custom task runner. +# +# Variable: AIRFLOW__CORE__TASK_RUNNER +# +# task_runner = StandardTaskRunner + +# If set, tasks without a ``run_as_user`` argument will be run with this user +# Can be used to de-elevate a sudo user running Airflow when executing tasks +# +# Variable: AIRFLOW__CORE__DEFAULT_IMPERSONATION +# +# default_impersonation = + +# What security module to use (for example kerberos) +# +# Variable: AIRFLOW__CORE__SECURITY +# +# security = + +# Turn unit test mode on (overwrites many configuration options with test +# values at runtime) +# +# Variable: AIRFLOW__CORE__UNIT_TEST_MODE +# +# unit_test_mode = False + +# Whether to enable pickling for xcom (note that this is insecure and allows for +# RCE exploits). +# +# Variable: AIRFLOW__CORE__ENABLE_XCOM_PICKLING +# +enable_xcom_pickling = False + +# What classes can be imported during deserialization. This is a multi line value. +# The individual items will be parsed as regexp. Python built-in classes (like dict) +# are always allowed. Bare "." will be replaced so you can set airflow.* . +# +# Variable: AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES +# +# allowed_deserialization_classes = airflow\..* + +# When a task is killed forcefully, this is the amount of time in seconds that +# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED +# +# Variable: AIRFLOW__CORE__KILLED_TASK_CLEANUP_TIME +# +killed_task_cleanup_time = 60 + +# Whether to override params with dag_run.conf. If you pass some key-value pairs +# through ``airflow dags backfill -c`` or +# ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. +# +# Variable: AIRFLOW__CORE__DAG_RUN_CONF_OVERRIDES_PARAMS +# +dag_run_conf_overrides_params = True + +# If enabled, Airflow will only scan files containing both ``DAG`` and ``airflow`` (case-insensitive). +# +# Variable: AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE +# +dag_discovery_safe_mode = True + +# The pattern syntax used in the ".airflowignore" files in the DAG directories. Valid values are +# ``regexp`` or ``glob``. +# +# Variable: AIRFLOW__CORE__DAG_IGNORE_FILE_SYNTAX +# +# dag_ignore_file_syntax = regexp + +# The number of retries each task is going to have by default. Can be overridden at dag or task level. +# +# Variable: AIRFLOW__CORE__DEFAULT_TASK_RETRIES +# +default_task_retries = 2 + +# The number of seconds each task is going to wait by default between retries. Can be overridden at +# dag or task level. +# +# Variable: AIRFLOW__CORE__DEFAULT_TASK_RETRY_DELAY +# +default_task_retry_delay = 300 + +# The maximum delay (in seconds) each task is going to wait by default between retries. +# This is a global setting and cannot be overridden at task or DAG level. +# +# Variable: AIRFLOW__CORE__MAX_TASK_RETRY_DELAY +# +max_task_retry_delay = 86400 + +# The weighting method used for the effective total priority weight of the task +# +# Variable: AIRFLOW__CORE__DEFAULT_TASK_WEIGHT_RULE +# +default_task_weight_rule = downstream + +# The default task execution_timeout value for the operators. Expected an integer value to +# be passed into timedelta as seconds. If not specified, then the value is considered as None, +# meaning that the operators are never timed out by default. +# +# Variable: AIRFLOW__CORE__DEFAULT_TASK_EXECUTION_TIMEOUT +# +# default_task_execution_timeout = + +# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. +# +# Variable: AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL +# +# min_serialized_dag_update_interval = 30 + +# If True, serialized DAGs are compressed before writing to DB. +# Note: this will disable the DAG dependencies view +# +# Variable: AIRFLOW__CORE__COMPRESS_SERIALIZED_DAGS +# +# compress_serialized_dags = False + +# Fetching serialized DAG can not be faster than a minimum interval to reduce database +# read rate. This config controls when your DAGs are updated in the Webserver +# +# Variable: AIRFLOW__CORE__MIN_SERIALIZED_DAG_FETCH_INTERVAL +# +# min_serialized_dag_fetch_interval = 10 + +# Maximum number of Rendered Task Instance Fields (Template Fields) per task to store +# in the Database. +# All the template_fields for each of Task Instance are stored in the Database. +# Keeping this number small may cause an error when you try to view ``Rendered`` tab in +# TaskInstance view for older tasks. +# +# Variable: AIRFLOW__CORE__MAX_NUM_RENDERED_TI_FIELDS_PER_TASK +# +# max_num_rendered_ti_fields_per_task = 30 + +# On each dagrun check against defined SLAs +# +# Variable: AIRFLOW__CORE__CHECK_SLAS +# +# check_slas = True + +# Path to custom XCom class that will be used to store and resolve operators results +# +# Example: xcom_backend = path.to.CustomXCom +# +# Variable: AIRFLOW__CORE__XCOM_BACKEND +# +xcom_backend = airflow.models.xcom.BaseXCom + +# By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``, +# if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module. +# +# Variable: AIRFLOW__CORE__LAZY_LOAD_PLUGINS +# +lazy_load_plugins = True + +# By default Airflow providers are lazily-discovered (discovery and imports happen only when required). +# Set it to False, if you want to discover providers whenever 'airflow' is invoked via cli or +# loaded from module. +# +# Variable: AIRFLOW__CORE__LAZY_DISCOVER_PROVIDERS +# +lazy_discover_providers = True + +# Hide sensitive Variables or Connection extra json keys from UI and task logs when set to True +# +# (Connection passwords are always hidden in logs) +# +# Variable: AIRFLOW__CORE__HIDE_SENSITIVE_VAR_CONN_FIELDS +# +# hide_sensitive_var_conn_fields = True + +# A comma-separated list of extra sensitive keywords to look for in variables names or connection's +# extra JSON. +# +# Variable: AIRFLOW__CORE__SENSITIVE_VAR_CONN_NAMES +# +# sensitive_var_conn_names = + +# Task Slot counts for ``default_pool``. This setting would not have any effect in an existing +# deployment where the ``default_pool`` is already created. For existing deployments, users can +# change the number of slots using Webserver, API or the CLI +# +# Variable: AIRFLOW__CORE__DEFAULT_POOL_TASK_SLOT_COUNT +# +# default_pool_task_slot_count = 128 + +# The maximum list/dict length an XCom can push to trigger task mapping. If the pushed list/dict has a +# length exceeding this value, the task pushing the XCom will be failed automatically to prevent the +# mapped tasks from clogging the scheduler. +# +# Variable: AIRFLOW__CORE__MAX_MAP_LENGTH +# +# max_map_length = 1024 + +# The default umask to use for process when run in daemon mode (scheduler, worker, etc.) +# +# This controls the file-creation mode mask which determines the initial value of file permission bits +# for newly created files. +# +# This value is treated as an octal-integer. +# +# Variable: AIRFLOW__CORE__DAEMON_UMASK +# +# daemon_umask = 0o077 + +# Class to use as dataset manager. +# +# Example: dataset_manager_class = airflow.datasets.manager.DatasetManager +# +# Variable: AIRFLOW__CORE__DATASET_MANAGER_CLASS +# +# dataset_manager_class = + +# Kwargs to supply to dataset manager. +# +# Example: dataset_manager_kwargs = {"some_param": "some_value"} +# +# Variable: AIRFLOW__CORE__DATASET_MANAGER_KWARGS +# +# dataset_manager_kwargs = + +# (experimental) Whether components should use Airflow Internal API for DB connectivity. +# +# Variable: AIRFLOW__CORE__DATABASE_ACCESS_ISOLATION +# +# database_access_isolation = False + +# (experimental) Airflow Internal API url. Only used if [core] database_access_isolation is True. +# +# Example: internal_api_url = http://localhost:8080 +# +# Variable: AIRFLOW__CORE__INTERNAL_API_URL +# +# internal_api_url = + +# The ability to allow testing connections across Airflow UI, API and CLI. +# Supported options: Disabled, Enabled, Hidden. Default: Disabled +# Disabled - Disables the test connection functionality and disables the Test Connection button in UI. +# Enabled - Enables the test connection functionality and shows the Test Connection button in UI. +# Hidden - Disables the test connection functionality and hides the Test Connection button in UI. +# Before setting this to Enabled, make sure that you review the users who are able to add/edit +# connections and ensure they are trusted. Connection testing can be done maliciously leading to +# undesired and insecure outcomes. For more information on capabilities of users, see the documentation: +# https://airflow.apache.org/docs/apache-airflow/stable/security/security_model.html#capabilities-of-authenticated-ui-users +# +# Variable: AIRFLOW__CORE__TEST_CONNECTION +# +# test_connection = Disabled + +[database] +# The SqlAlchemy connection string to the metadata database. +# SqlAlchemy supports many different database engines. +# More information here: +# http://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html#database-uri +sql_alchemy_conn = postgresql+psycopg2://airflow:airflow@postgres/airflow + +# Path to the ``alembic.ini`` file. You can either provide the file path relative +# to the Airflow home directory or the absolute path if it is located elsewhere. +# +# Variable: AIRFLOW__DATABASE__ALEMBIC_INI_FILE_PATH +# +# alembic_ini_file_path = alembic.ini + +# Extra engine specific keyword args passed to SQLAlchemy's create_engine, as a JSON-encoded value +# +# Example: sql_alchemy_engine_args = {"arg1": True} +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_ENGINE_ARGS +# +# sql_alchemy_engine_args = + +# The encoding for the databases +# +# Variable: AIRFLOW__DATABASE__SQL_ENGINE_ENCODING +# +# sql_engine_encoding = utf-8 + +# Collation for ``dag_id``, ``task_id``, ``key``, ``external_executor_id`` columns +# in case they have different encoding. +# By default this collation is the same as the database collation, however for ``mysql`` and ``mariadb`` +# the default is ``utf8mb3_bin`` so that the index sizes of our index keys will not exceed +# the maximum size of allowed index when collation is set to ``utf8mb4`` variant +# (see https://github.com/apache/airflow/pull/17603#issuecomment-901121618). +# +# Variable: AIRFLOW__DATABASE__SQL_ENGINE_COLLATION_FOR_IDS +# +# sql_engine_collation_for_ids = + +# If SqlAlchemy should pool database connections. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_ENABLED +# +# sql_alchemy_pool_enabled = True + +# The SqlAlchemy pool size is the maximum number of database connections +# in the pool. 0 indicates no limit. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE +# +# sql_alchemy_pool_size = 5 + +# The maximum overflow size of the pool. +# When the number of checked-out connections reaches the size set in pool_size, +# additional connections will be returned up to this limit. +# When those additional connections are returned to the pool, they are disconnected and discarded. +# It follows then that the total number of simultaneous connections the pool will allow +# is pool_size + max_overflow, +# and the total number of "sleeping" connections the pool will allow is pool_size. +# max_overflow can be set to ``-1`` to indicate no overflow limit; +# no limit will be placed on the total number of concurrent connections. Defaults to ``10``. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW +# +# sql_alchemy_max_overflow = 10 + +# The SqlAlchemy pool recycle is the number of seconds a connection +# can be idle in the pool before it is invalidated. This config does +# not apply to sqlite. If the number of DB connections is ever exceeded, +# a lower config value will allow the system to recover faster. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_RECYCLE +# +# sql_alchemy_pool_recycle = 1800 + +# Check connection at the start of each connection pool checkout. +# Typically, this is a simple statement like "SELECT 1". +# More information here: +# https://docs.sqlalchemy.org/en/14/core/pooling.html#disconnect-handling-pessimistic +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_PRE_PING +# +# sql_alchemy_pool_pre_ping = True + +# The schema to use for the metadata database. +# SqlAlchemy supports databases with the concept of multiple schemas. +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_SCHEMA +# +# sql_alchemy_schema = + +# Import path for connect args in SqlAlchemy. Defaults to an empty dict. +# This is useful when you want to configure db engine args that SqlAlchemy won't parse +# in connection string. +# See https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine.params.connect_args +# +# Example: sql_alchemy_connect_args = {"timeout": 30} +# +# Variable: AIRFLOW__DATABASE__SQL_ALCHEMY_CONNECT_ARGS +# +# sql_alchemy_connect_args = + +# Whether to load the default connections that ship with Airflow when ``airflow db init`` is called. +# It's good to get started, but you probably want to set this to ``False`` in a production environment. +# +# Variable: AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS +# +# load_default_connections = True + +# Number of times the code should be retried in case of DB Operational Errors. +# Not all transactions will be retried as it can cause undesired state. +# Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. +# +# Variable: AIRFLOW__DATABASE__MAX_DB_RETRIES +# +# max_db_retries = 3 + +# Whether to run alembic migrations during Airflow start up. Sometimes this operation can be expensive, +# and the users can assert the correct version through other means (e.g. through a Helm chart). +# Accepts "True" or "False". +# +# Variable: AIRFLOW__DATABASE__CHECK_MIGRATIONS +# +# check_migrations = True + +[logging] +# The folder where airflow should store its log files. +# This path must be absolute. +# There are a few existing configurations that assume this is set to the default. +# If you choose to override this you may need to update the dag_processor_manager_log_location and +# child_process_log_directory settings as well. +# +# Variable: AIRFLOW__LOGGING__BASE_LOG_FOLDER +# +base_log_folder = $AIRFLOW_HOME/logs + +# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. +# Set this to True if you want to enable remote logging. +# +# Variable: AIRFLOW__LOGGING__REMOTE_LOGGING +# +# remote_logging = False + +# Users must supply an Airflow connection id that provides access to the storage +# location. Depending on your remote logging service, this may only be used for +# reading logs, not writing them. +# +# Variable: AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID +# +# remote_log_conn_id = + +# Whether the local log files for GCS, S3, WASB and OSS remote logging should be deleted after +# they are uploaded to the remote location. +# +# Variable: AIRFLOW__LOGGING__DELETE_LOCAL_LOGS +# +# delete_local_logs = False + +# Path to Google Credential JSON file. If omitted, authorization based on `the Application Default +# Credentials +# `__ will +# be used. +# +# Variable: AIRFLOW__LOGGING__GOOGLE_KEY_PATH +# +# google_key_path = + +# Storage bucket URL for remote logging +# S3 buckets should start with "s3://" +# Cloudwatch log groups should start with "cloudwatch://" +# GCS buckets should start with "gs://" +# WASB buckets should start with "wasb" just to help Airflow select correct handler +# Stackdriver logs should start with "stackdriver://" +# +# Variable: AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER +# +# remote_base_log_folder = + +# The remote_task_handler_kwargs param is loaded into a dictionary and passed to __init__ of remote +# task handler and it overrides the values provided by Airflow config. For example if you set +# `delete_local_logs=False` and you provide ``{"delete_local_copy": true}``, then the local +# log files will be deleted after they are uploaded to remote location. +# +# Example: remote_task_handler_kwargs = {"delete_local_copy": true} +# +# Variable: AIRFLOW__LOGGING__REMOTE_TASK_HANDLER_KWARGS +# +# remote_task_handler_kwargs = + +# Use server-side encryption for logs stored in S3 +# +# Variable: AIRFLOW__LOGGING__ENCRYPT_S3_LOGS +# +# encrypt_s3_logs = False + +# Logging level. +# +# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. +# +# Variable: AIRFLOW__LOGGING__LOGGING_LEVEL +# +logging_level = INFO + +# Logging level for celery. If not set, it uses the value of logging_level +# +# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. +# +# Variable: AIRFLOW__LOGGING__CELERY_LOGGING_LEVEL +# +# celery_logging_level = + +# Logging level for Flask-appbuilder UI. +# +# Supported values: ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO``, ``DEBUG``. +# +# Variable: AIRFLOW__LOGGING__FAB_LOGGING_LEVEL +# +# fab_logging_level = WARNING + +# Logging class +# Specify the class that will specify the logging configuration +# This class has to be on the python classpath +# +# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG +# +# Variable: AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS +# +# logging_config_class = + +# Flag to enable/disable Colored logs in Console +# Colour the logs when the controlling terminal is a TTY. +# +# Variable: AIRFLOW__LOGGING__COLORED_CONSOLE_LOG +# +colored_console_log = True + +# Log format for when Colored logs is enabled +# +# Variable: AIRFLOW__LOGGING__COLORED_LOG_FORMAT +# +# colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s + +# +# Variable: AIRFLOW__LOGGING__COLORED_FORMATTER_CLASS +# +# colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter + +# Format of Log line +# +# Variable: AIRFLOW__LOGGING__LOG_FORMAT +# +# log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s + +# +# Variable: AIRFLOW__LOGGING__SIMPLE_LOG_FORMAT +# +# simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s + +# Where to send dag parser logs. If "file", logs are sent to log files defined by child_process_log_directory. +# +# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_LOG_TARGET +# +# dag_processor_log_target = file + +# Format of Dag Processor Log line +# +# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_LOG_FORMAT +# +# dag_processor_log_format = [%%(asctime)s] [SOURCE:DAG_PROCESSOR] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s + +# +# Variable: AIRFLOW__LOGGING__LOG_FORMATTER_CLASS +# +# log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware + +# An import path to a function to add adaptations of each secret added with +# `airflow.utils.log.secrets_masker.mask_secret` to be masked in log messages. The given function +# is expected to require a single parameter: the secret to be adapted. It may return a +# single adaptation of the secret or an iterable of adaptations to each be masked as secrets. +# The original secret will be masked as well as any adaptations returned. +# +# Example: secret_mask_adapter = urllib.parse.quote +# +# Variable: AIRFLOW__LOGGING__SECRET_MASK_ADAPTER +# +# secret_mask_adapter = + +# Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter +# +# Example: task_log_prefix_template = {ti.dag_id}-{ti.task_id}-{execution_date}-{try_number} +# +# Variable: AIRFLOW__LOGGING__TASK_LOG_PREFIX_TEMPLATE +# +# task_log_prefix_template = + +# Formatting for how airflow generates file names/paths for each task run. +# +# Variable: AIRFLOW__LOGGING__LOG_FILENAME_TEMPLATE +# +# log_filename_template = dag_id={{ ti.dag_id }}/run_id={{ ti.run_id }}/task_id={{ ti.task_id }}/{%% if ti.map_index >= 0 %%}map_index={{ ti.map_index }}/{%% endif %%}attempt={{ try_number }}.log + +# Formatting for how airflow generates file names for log +# +# Variable: AIRFLOW__LOGGING__LOG_PROCESSOR_FILENAME_TEMPLATE +# +# log_processor_filename_template = {{ filename }}.log + +# Full path of dag_processor_manager logfile. +# +# Variable: AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION +# +# dag_processor_manager_log_location = $AIRFLOW_HOME/logs/dag_processor_manager/dag_processor_manager.log + +# Name of handler to read task instance logs. +# Defaults to use ``task`` handler. +# +# Variable: AIRFLOW__LOGGING__TASK_LOG_READER +# +# task_log_reader = task + +# A comma\-separated list of third-party logger names that will be configured to print messages to +# consoles\. +# +# Example: extra_logger_names = connexion,sqlalchemy +# +# Variable: AIRFLOW__LOGGING__EXTRA_LOGGER_NAMES +# +# extra_logger_names = + +# When you start an airflow worker, airflow starts a tiny web server +# subprocess to serve the workers local log files to the airflow main +# web server, who then builds pages and sends them to users. This defines +# the port on which the logs are served. It needs to be unused, and open +# visible from the main web server to connect into the workers. +# +# Variable: AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT +# +# worker_log_server_port = 8793 + +# Port to serve logs from for triggerer. See worker_log_server_port description +# for more info. +# +# Variable: AIRFLOW__LOGGING__TRIGGER_LOG_SERVER_PORT +# +# trigger_log_server_port = 8794 + +# We must parse timestamps to interleave logs between trigger and task. To do so, +# we need to parse timestamps in log files. In case your log format is non-standard, +# you may provide import path to callable which takes a string log line and returns +# the timestamp (datetime.datetime compatible). +# +# Example: interleave_timestamp_parser = path.to.my_func +# +# Variable: AIRFLOW__LOGGING__INTERLEAVE_TIMESTAMP_PARSER +# +# interleave_timestamp_parser = + +# Permissions in the form or of octal string as understood by chmod. The permissions are important +# when you use impersonation, when logs are written by a different user than airflow. The most secure +# way of configuring it in this case is to add both users to the same group and make it the default +# group of both users. Group-writeable logs are default in airflow, but you might decide that you are +# OK with having the logs other-writeable, in which case you should set it to `0o777`. You might +# decide to add more security if you do not use impersonation and change it to `0o755` to make it +# only owner-writeable. You can also make it just readable only for owner by changing it to `0o700` if +# all the access (read/write) for your logs happens from the same user. +# +# Example: file_task_handler_new_folder_permissions = 0o775 +# +# Variable: AIRFLOW__LOGGING__FILE_TASK_HANDLER_NEW_FOLDER_PERMISSIONS +# +# file_task_handler_new_folder_permissions = 0o775 + +# Permissions in the form or of octal string as understood by chmod. The permissions are important +# when you use impersonation, when logs are written by a different user than airflow. The most secure +# way of configuring it in this case is to add both users to the same group and make it the default +# group of both users. Group-writeable logs are default in airflow, but you might decide that you are +# OK with having the logs other-writeable, in which case you should set it to `0o666`. You might +# decide to add more security if you do not use impersonation and change it to `0o644` to make it +# only owner-writeable. You can also make it just readable only for owner by changing it to `0o600` if +# all the access (read/write) for your logs happens from the same user. +# +# Example: file_task_handler_new_file_permissions = 0o664 +# +# Variable: AIRFLOW__LOGGING__FILE_TASK_HANDLER_NEW_FILE_PERMISSIONS +# +# file_task_handler_new_file_permissions = 0o664 + +# By default Celery sends all logs into stderr. +# If enabled any previous logging handlers will get *removed*. +# With this option AirFlow will create new handlers +# and send low level logs like INFO and WARNING to stdout, +# while sending higher severity logs to stderr. +# +# Variable: AIRFLOW__LOGGING__CELERY_STDOUT_STDERR_SEPARATION +# +# celery_stdout_stderr_separation = False + +[metrics] +# StatsD (https://github.com/etsy/statsd) integration settings. + +# If you want to avoid emitting all the available metrics, you can configure an +# allow list of prefixes (comma separated) to send only the metrics that start +# with the elements of the list (e.g: "scheduler,executor,dagrun") +# +# Variable: AIRFLOW__METRICS__METRICS_ALLOW_LIST +# +# metrics_allow_list = + +# If you want to avoid emitting all the available metrics, you can configure a +# block list of prefixes (comma separated) to filter out metrics that start with +# the elements of the list (e.g: "scheduler,executor,dagrun"). +# If metrics_allow_list and metrics_block_list are both configured, metrics_block_list is ignored. +# +# Variable: AIRFLOW__METRICS__METRICS_BLOCK_LIST +# +# metrics_block_list = + +# Enables sending metrics to StatsD. +# +# Variable: AIRFLOW__METRICS__STATSD_ON +# +# statsd_on = False + +# +# Variable: AIRFLOW__METRICS__STATSD_HOST +# +# statsd_host = localhost + +# +# Variable: AIRFLOW__METRICS__STATSD_PORT +# +# statsd_port = 8125 + +# +# Variable: AIRFLOW__METRICS__STATSD_PREFIX +# +# statsd_prefix = airflow + +# A function that validate the StatsD stat name, apply changes to the stat name if necessary and return +# the transformed stat name. +# +# The function should have the following signature: +# def func_name(stat_name: str) -> str: +# +# Variable: AIRFLOW__METRICS__STAT_NAME_HANDLER +# +# stat_name_handler = + +# To enable datadog integration to send airflow metrics. +# +# Variable: AIRFLOW__METRICS__STATSD_DATADOG_ENABLED +# +# statsd_datadog_enabled = False + +# List of datadog tags attached to all metrics(e.g: key1:value1,key2:value2) +# +# Variable: AIRFLOW__METRICS__STATSD_DATADOG_TAGS +# +# statsd_datadog_tags = + +# Set to False to disable metadata tags for some of the emitted metrics +# +# Variable: AIRFLOW__METRICS__STATSD_DATADOG_METRICS_TAGS +# +# statsd_datadog_metrics_tags = True + +# If you want to utilise your own custom StatsD client set the relevant +# module path below. +# Note: The module path must exist on your PYTHONPATH for Airflow to pick it up +# +# Variable: AIRFLOW__METRICS__STATSD_CUSTOM_CLIENT_PATH +# +# statsd_custom_client_path = + +# If you want to avoid sending all the available metrics tags to StatsD, +# you can configure a block list of prefixes (comma separated) to filter out metric tags +# that start with the elements of the list (e.g: "job_id,run_id") +# +# Example: statsd_disabled_tags = job_id,run_id,dag_id,task_id +# +# Variable: AIRFLOW__METRICS__STATSD_DISABLED_TAGS +# +# statsd_disabled_tags = job_id,run_id + +# To enable sending Airflow metrics with StatsD-Influxdb tagging convention. +# +# Variable: AIRFLOW__METRICS__STATSD_INFLUXDB_ENABLED +# +# statsd_influxdb_enabled = False + +# Enables sending metrics to OpenTelemetry. +# +# Variable: AIRFLOW__METRICS__OTEL_ON +# +# otel_on = False + +# +# Variable: AIRFLOW__METRICS__OTEL_HOST +# +# otel_host = localhost + +# +# Variable: AIRFLOW__METRICS__OTEL_PORT +# +# otel_port = 8889 + +# +# Variable: AIRFLOW__METRICS__OTEL_PREFIX +# +# otel_prefix = airflow + +# +# Variable: AIRFLOW__METRICS__OTEL_INTERVAL_MILLISECONDS +# +# otel_interval_milliseconds = 60000 + +# If True, all metrics are also emitted to the console. Defaults to False. +# +# Variable: AIRFLOW__METRICS__OTEL_DEBUGGING_ON +# +# otel_debugging_on = False + +# If True, SSL will be enabled. Defaults to False. +# To establish an HTTPS connection to the OpenTelemetry collector, +# you need to configure the SSL certificate and key within the OpenTelemetry collector's +# config.yml file. +# +# Variable: AIRFLOW__METRICS__OTEL_SSL_ACTIVE +# +# otel_ssl_active = False + +[secrets] +# Full class name of secrets backend to enable (will precede env vars and metastore in search path) +# +# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend +# +# Variable: AIRFLOW__SECRETS__BACKEND +# +# backend = + +# The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. +# See documentation for the secrets backend you are using. JSON is expected. +# Example for AWS Systems Manager ParameterStore: +# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` +# +# Variable: AIRFLOW__SECRETS__BACKEND_KWARGS +# +# backend_kwargs = + +# .. note:: |experimental| +# +# Enables local caching of Variables, when parsing DAGs only. +# Using this option can make dag parsing faster if Variables are used in top level code, at the expense +# of longer propagation time for changes. +# Please note that this cache concerns only the DAG parsing step. There is no caching in place when DAG +# tasks are run. +# +# Variable: AIRFLOW__SECRETS__USE_CACHE +# +# use_cache = False + +# .. note:: |experimental| +# +# When the cache is enabled, this is the duration for which we consider an entry in the cache to be +# valid. Entries are refreshed if they are older than this many seconds. +# It means that when the cache is enabled, this is the maximum amount of time you need to wait to see a +# Variable change take effect. +# +# Variable: AIRFLOW__SECRETS__CACHE_TTL_SECONDS +# +# cache_ttl_seconds = 900 + +[cli] +# In what way should the cli access the API. The LocalClient will use the +# database directly, while the json_client will use the api running on the +# webserver +# +# Variable: AIRFLOW__CLI__API_CLIENT +# +# api_client = airflow.api.client.local_client + +# If you set web_server_url_prefix, do NOT forget to append it here, ex: +# ``endpoint_url = http://localhost:8080/myroot`` +# So api will look like: ``http://localhost:8080/myroot/api/experimental/...`` +# +# Variable: AIRFLOW__CLI__ENDPOINT_URL +# +# endpoint_url = http://localhost:8080 + +[debug] +# Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first +# failed task. Helpful for debugging purposes. +# +# Variable: AIRFLOW__DEBUG__FAIL_FAST +# +# fail_fast = False + +[api] +# Enables the deprecated experimental API. Please note that these APIs do not have access control. +# The authenticated user has full access. +# +# .. warning:: +# +# This `Experimental REST API `__ is +# deprecated since version 2.0. Please consider using +# `the Stable REST API `__. +# For more information on migration, see +# `RELEASE_NOTES.rst `_ +# +# Variable: AIRFLOW__API__ENABLE_EXPERIMENTAL_API +# +# enable_experimental_api = False + +# Comma separated list of auth backends to authenticate users of the API. See +# https://airflow.apache.org/docs/apache-airflow/stable/security/api.html for possible values. +# ("airflow.api.auth.backend.default" allows all requests for historic reasons) +# +# Variable: AIRFLOW__API__AUTH_BACKENDS +# +# auth_backends = airflow.api.auth.backend.session + +# Used to set the maximum page limit for API requests. If limit passed as param +# is greater than maximum page limit, it will be ignored and maximum page limit value +# will be set as the limit +# +# Variable: AIRFLOW__API__MAXIMUM_PAGE_LIMIT +# +# maximum_page_limit = 100 + +# Used to set the default page limit when limit param is zero or not provided in API +# requests. Otherwise if positive integer is passed in the API requests as limit, the +# smallest number of user given limit or maximum page limit is taken as limit. +# +# Variable: AIRFLOW__API__FALLBACK_PAGE_LIMIT +# +# fallback_page_limit = 100 + +# The intended audience for JWT token credentials used for authorization. This value must match on the client and server sides. If empty, audience will not be tested. +# +# Example: google_oauth2_audience = project-id-random-value.apps.googleusercontent.com +# +# Variable: AIRFLOW__API__GOOGLE_OAUTH2_AUDIENCE +# +# google_oauth2_audience = + +# Path to Google Cloud Service Account key file (JSON). If omitted, authorization based on +# `the Application Default Credentials +# `__ will +# be used. +# +# Example: google_key_path = /files/service-account-json +# +# Variable: AIRFLOW__API__GOOGLE_KEY_PATH +# +# google_key_path = + +# Used in response to a preflight request to indicate which HTTP +# headers can be used when making the actual request. This header is +# the server side response to the browser's +# Access-Control-Request-Headers header. +# +# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_HEADERS +# +# access_control_allow_headers = + +# Specifies the method or methods allowed when accessing the resource. +# +# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_METHODS +# +# access_control_allow_methods = + +# Indicates whether the response can be shared with requesting code from the given origins. +# Separate URLs with space. +# +# Variable: AIRFLOW__API__ACCESS_CONTROL_ALLOW_ORIGINS +# +# access_control_allow_origins = + +# Indicates whether the *xcomEntries* endpoint supports the *deserialize* +# flag. If set to False, setting this flag in a request would result in a +# 400 Bad Request error. +# +# Variable: AIRFLOW__API__ENABLE_XCOM_DESERIALIZE_SUPPORT +# +# enable_xcom_deserialize_support = False + +[lineage] +# what lineage backend to use +# +# Variable: AIRFLOW__LINEAGE__BACKEND +# +# backend = + +[operators] +# The default owner assigned to each new operator, unless +# provided explicitly or passed via ``default_args`` +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_OWNER +# +# default_owner = airflow + +# The default value of attribute "deferrable" in operators and sensors. +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_DEFERRABLE +# +# default_deferrable = false + +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_CPUS +# +# default_cpus = 1 + +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_RAM +# +# default_ram = 512 + +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_DISK +# +# default_disk = 512 + +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_GPUS +# +# default_gpus = 0 + +# Default queue that tasks get assigned to and that worker listen on. +# +# Variable: AIRFLOW__OPERATORS__DEFAULT_QUEUE +# +# default_queue = default + +# Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator. +# If set to False, an exception will be thrown, otherwise only the console message will be displayed. +# +# Variable: AIRFLOW__OPERATORS__ALLOW_ILLEGAL_ARGUMENTS +# +# allow_illegal_arguments = False + +[webserver] +# The message displayed when a user attempts to execute actions beyond their authorised privileges. +# +# Variable: AIRFLOW__WEBSERVER__ACCESS_DENIED_MESSAGE +# +# access_denied_message = Access is Denied + +# Path of webserver config file used for configuring the webserver parameters +# +# Variable: AIRFLOW__WEBSERVER__CONFIG_FILE +# +config_file = $AIRFLOW_HOME/webserver_config.py + +# The base url of your website as airflow cannot guess what domain or +# cname you are using. This is used in automated emails that +# airflow sends to point links to the right web server +# +# Variable: AIRFLOW__WEBSERVER__BASE_URL +# +# base_url = http://localhost:8080 + +# Default timezone to display all dates in the UI, can be UTC, system, or +# any IANA timezone string (e.g. Europe/Amsterdam). If left empty the +# default value of core/default_timezone will be used +# +# Example: default_ui_timezone = America/New_York +# +# Variable: AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE +# +# default_ui_timezone = UTC + +# The ip specified when starting the web server +# +# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_HOST +# +# web_server_host = 0.0.0.0 + +# The port on which to run the web server +# +# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_PORT +# +# web_server_port = 8080 + +# Paths to the SSL certificate and key for the web server. When both are +# provided SSL will be enabled. This does not change the web server port. +# +# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT +# +# web_server_ssl_cert = + +# Paths to the SSL certificate and key for the web server. When both are +# provided SSL will be enabled. This does not change the web server port. +# +# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY +# +# web_server_ssl_key = + +# The type of backend used to store web session data, can be `database` or `securecookie`. For the +# `database` backend, sessions are store in the database (in `session` table) and they can be +# managed there (for example when you reset password of the user, all sessions for that user are +# deleted). For the `securecookie` backend, sessions are stored in encrypted cookies on the client +# side. The `securecookie` mechanism is 'lighter' than database backend, but sessions are not deleted +# when you reset password of the user, which means that other than waiting for expiry time, the only +# way to invalidate all sessions for a user is to change secret_key and restart webserver (which +# also invalidates and logs out all other user's sessions). +# +# When you are using `database` backend, make sure to keep your database session table small +# by periodically running `airflow db clean --table session` command, especially if you have +# automated API calls that will create a new session for each call rather than reuse the sessions +# stored in browser cookies. +# +# Example: session_backend = securecookie +# +# Variable: AIRFLOW__WEBSERVER__SESSION_BACKEND +# +# session_backend = database + +# Number of seconds the webserver waits before killing gunicorn master that doesn't respond +# +# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT +# +# web_server_master_timeout = 120 + +# Number of seconds the gunicorn webserver waits before timing out on a worker +# +# Variable: AIRFLOW__WEBSERVER__WEB_SERVER_WORKER_TIMEOUT +# +# web_server_worker_timeout = 120 + +# Number of workers to refresh at a time. When set to 0, worker refresh is +# disabled. When nonzero, airflow periodically refreshes webserver workers by +# bringing up new ones and killing old ones. +# +# Variable: AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE +# +# worker_refresh_batch_size = 1 + +# Number of seconds to wait before refreshing a batch of workers. +# +# Variable: AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL +# +# worker_refresh_interval = 6000 + +# If set to True, Airflow will track files in plugins_folder directory. When it detects changes, +# then reload the gunicorn. If set to True, gunicorn starts without preloading, which is slower, uses +# more memory, and may cause race conditions. Avoid setting this to True in production. +# +# Variable: AIRFLOW__WEBSERVER__RELOAD_ON_PLUGIN_CHANGE +# +# reload_on_plugin_change = False + +# Secret key used to run your flask app. It should be as random as possible. However, when running +# more than 1 instances of webserver, make sure all of them use the same ``secret_key`` otherwise +# one of them will error with "CSRF session token is missing". +# The webserver key is also used to authorize requests to Celery workers when logs are retrieved. +# The token generated using the secret key has a short expiry time though - make sure that time on +# ALL the machines that you run airflow components on is synchronized (for example using ntpd) +# otherwise you might get "forbidden" errors when the logs are accessed. +# +# Variable: AIRFLOW__WEBSERVER__SECRET_KEY +# +secret_key = $AIRFLOW__WEBSERVER__SECRET_KEY + +# Number of workers to run the Gunicorn web server +# +# Variable: AIRFLOW__WEBSERVER__WORKERS +# +# workers = 4 + +# The worker class gunicorn should use. Choices include +# sync (default), eventlet, gevent. Note when using gevent you might also want to set the +# "_AIRFLOW_PATCH_GEVENT" environment variable to "1" to make sure gevent patching is done as +# early as possible. +# +# Variable: AIRFLOW__WEBSERVER__WORKER_CLASS +# +# worker_class = sync + +# Log files for the gunicorn webserver. '-' means log to stderr. +# +# Variable: AIRFLOW__WEBSERVER__ACCESS_LOGFILE +# +# access_logfile = - + +# Log files for the gunicorn webserver. '-' means log to stderr. +# +# Variable: AIRFLOW__WEBSERVER__ERROR_LOGFILE +# +# error_logfile = - + +# Access log format for gunicorn webserver. +# default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s" +# documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format +# +# Variable: AIRFLOW__WEBSERVER__ACCESS_LOGFORMAT +# +# access_logformat = + +# Expose the configuration file in the web server. Set to "non-sensitive-only" to show all values +# except those that have security implications. "True" shows all values. "False" hides the +# configuration completely. +# +# Variable: AIRFLOW__WEBSERVER__EXPOSE_CONFIG +# +# expose_config = False + +# Expose hostname in the web server +# +# Variable: AIRFLOW__WEBSERVER__EXPOSE_HOSTNAME +# +# expose_hostname = False + +# Expose stacktrace in the web server +# +# Variable: AIRFLOW__WEBSERVER__EXPOSE_STACKTRACE +# +# expose_stacktrace = False + +# Default DAG view. Valid values are: ``grid``, ``graph``, ``duration``, ``gantt``, ``landing_times`` +# +# Variable: AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW +# +# dag_default_view = grid + +# Default DAG orientation. Valid values are: +# ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT`` (Bottom->Top) +# +# Variable: AIRFLOW__WEBSERVER__DAG_ORIENTATION +# +# dag_orientation = LR + +# Sorting order in grid view. Valid values are: ``topological``, ``hierarchical_alphabetical`` +# +# Variable: AIRFLOW__WEBSERVER__GRID_VIEW_SORTING_ORDER +# +# grid_view_sorting_order = topological + +# The amount of time (in secs) webserver will wait for initial handshake +# while fetching logs from other worker machine +# +# Variable: AIRFLOW__WEBSERVER__LOG_FETCH_TIMEOUT_SEC +# +# log_fetch_timeout_sec = 5 + +# Time interval (in secs) to wait before next log fetching. +# +# Variable: AIRFLOW__WEBSERVER__LOG_FETCH_DELAY_SEC +# +# log_fetch_delay_sec = 2 + +# Distance away from page bottom to enable auto tailing. +# +# Variable: AIRFLOW__WEBSERVER__LOG_AUTO_TAILING_OFFSET +# +# log_auto_tailing_offset = 30 + +# Animation speed for auto tailing log display. +# +# Variable: AIRFLOW__WEBSERVER__LOG_ANIMATION_SPEED +# +# log_animation_speed = 1000 + +# By default, the webserver shows paused DAGs. Flip this to hide paused +# DAGs by default +# +# Variable: AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT +# +# hide_paused_dags_by_default = False + +# Consistent page size across all listing views in the UI +# +# Variable: AIRFLOW__WEBSERVER__PAGE_SIZE +# +# page_size = 100 + +# Define the color of navigation bar +# +# Variable: AIRFLOW__WEBSERVER__NAVBAR_COLOR +# +# navbar_color = #fff + +# Default dagrun to show in UI +# +# Variable: AIRFLOW__WEBSERVER__DEFAULT_DAG_RUN_DISPLAY_NUMBER +# +# default_dag_run_display_number = 25 + +# Enable werkzeug ``ProxyFix`` middleware for reverse proxy +# +# Variable: AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX +# +# enable_proxy_fix = False + +# Number of values to trust for ``X-Forwarded-For``. +# More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/ +# +# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_FOR +# +# proxy_fix_x_for = 1 + +# Number of values to trust for ``X-Forwarded-Proto`` +# +# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_PROTO +# +# proxy_fix_x_proto = 1 + +# Number of values to trust for ``X-Forwarded-Host`` +# +# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_HOST +# +# proxy_fix_x_host = 1 + +# Number of values to trust for ``X-Forwarded-Port`` +# +# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_PORT +# +# proxy_fix_x_port = 1 + +# Number of values to trust for ``X-Forwarded-Prefix`` +# +# Variable: AIRFLOW__WEBSERVER__PROXY_FIX_X_PREFIX +# +# proxy_fix_x_prefix = 1 + +# Set secure flag on session cookie +# +# Variable: AIRFLOW__WEBSERVER__COOKIE_SECURE +# +# cookie_secure = False + +# Set samesite policy on session cookie +# +# Variable: AIRFLOW__WEBSERVER__COOKIE_SAMESITE +# +# cookie_samesite = Lax + +# Default setting for wrap toggle on DAG code and TI log views. +# +# Variable: AIRFLOW__WEBSERVER__DEFAULT_WRAP +# +# default_wrap = False + +# Allow the UI to be rendered in a frame +# +# Variable: AIRFLOW__WEBSERVER__X_FRAME_ENABLED +# +# x_frame_enabled = True + +# Send anonymous user activity to your analytics tool +# choose from google_analytics, segment, or metarouter +# +# Variable: AIRFLOW__WEBSERVER__ANALYTICS_TOOL +# +# analytics_tool = + +# Unique ID of your account in the analytics tool +# +# Variable: AIRFLOW__WEBSERVER__ANALYTICS_ID +# +# analytics_id = + +# 'Recent Tasks' stats will show for old DagRuns if set +# +# Variable: AIRFLOW__WEBSERVER__SHOW_RECENT_STATS_FOR_COMPLETED_RUNS +# +# show_recent_stats_for_completed_runs = True + +# Update FAB permissions and sync security manager roles +# on webserver startup +# +# Variable: AIRFLOW__WEBSERVER__UPDATE_FAB_PERMS +# +# update_fab_perms = True + +# The UI cookie lifetime in minutes. User will be logged out from UI after +# ``session_lifetime_minutes`` of non-activity +# +# Variable: AIRFLOW__WEBSERVER__SESSION_LIFETIME_MINUTES +# +# session_lifetime_minutes = 43200 + +# Sets a custom page title for the DAGs overview page and site title for all pages +# +# Variable: AIRFLOW__WEBSERVER__INSTANCE_NAME +# +# instance_name = + +# Whether the custom page title for the DAGs overview page contains any Markup language +# +# Variable: AIRFLOW__WEBSERVER__INSTANCE_NAME_HAS_MARKUP +# +# instance_name_has_markup = False + +# How frequently, in seconds, the DAG data will auto-refresh in graph or grid view +# when auto-refresh is turned on +# +# Variable: AIRFLOW__WEBSERVER__AUTO_REFRESH_INTERVAL +# +# auto_refresh_interval = 3 + +# Boolean for displaying warning for publicly viewable deployment +# +# Variable: AIRFLOW__WEBSERVER__WARN_DEPLOYMENT_EXPOSURE +# +# warn_deployment_exposure = True + +# Comma separated string of view events to exclude from dag audit view. +# All other events will be added minus the ones passed here. +# The audit logs in the db will not be affected by this parameter. +# +# Variable: AIRFLOW__WEBSERVER__AUDIT_VIEW_EXCLUDED_EVENTS +# +# audit_view_excluded_events = gantt,landing_times,tries,duration,calendar,graph,grid,tree,tree_data + +# Comma separated string of view events to include in dag audit view. +# If passed, only these events will populate the dag audit view. +# The audit logs in the db will not be affected by this parameter. +# +# Example: audit_view_included_events = dagrun_cleared,failed +# +# Variable: AIRFLOW__WEBSERVER__AUDIT_VIEW_INCLUDED_EVENTS +# +# audit_view_included_events = + +# Boolean for running SwaggerUI in the webserver. +# +# Variable: AIRFLOW__WEBSERVER__ENABLE_SWAGGER_UI +# +# enable_swagger_ui = True + +# Boolean for running Internal API in the webserver. +# +# Variable: AIRFLOW__WEBSERVER__RUN_INTERNAL_API +# +# run_internal_api = False + +# Boolean for enabling rate limiting on authentication endpoints. +# +# Variable: AIRFLOW__WEBSERVER__AUTH_RATE_LIMITED +# +# auth_rate_limited = True + +# Rate limit for authentication endpoints. +# +# Variable: AIRFLOW__WEBSERVER__AUTH_RATE_LIMIT +# +# auth_rate_limit = 5 per 40 second + +# The caching algorithm used by the webserver. Must be a valid hashlib function name. +# +# Example: caching_hash_method = sha256 +# +# Variable: AIRFLOW__WEBSERVER__CACHING_HASH_METHOD +# +# caching_hash_method = md5 + +# Behavior of the trigger DAG run button for DAGs without params. False to skip and trigger +# without displaying a form to add a dag_run.conf, True to always display the form. +# The form is displayed always if parameters are defined. +# +# Variable: AIRFLOW__WEBSERVER__SHOW_TRIGGER_FORM_IF_NO_PARAMS +# +# show_trigger_form_if_no_params = False + +[email] +# Configuration email backend and whether to +# send email alerts on retry or failure + +# Email backend to use +# +# Variable: AIRFLOW__EMAIL__EMAIL_BACKEND +# +email_backend = airflow.utils.email.send_email_smtp + +# Email connection to use +# +# Variable: AIRFLOW__EMAIL__EMAIL_CONN_ID +# +email_conn_id = smtp_default + +# Whether email alerts should be sent when a task is retried +# +# Variable: AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_RETRY +# +default_email_on_retry = True + +# Whether email alerts should be sent when a task failed +# +# Variable: AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_FAILURE +# +default_email_on_failure = True + +# File that will be used as the template for Email subject (which will be rendered using Jinja2). +# If not set, Airflow uses a base template. +# +# Example: subject_template = /path/to/my_subject_template_file +# +# Variable: AIRFLOW__EMAIL__SUBJECT_TEMPLATE +# +# subject_template = + +# File that will be used as the template for Email content (which will be rendered using Jinja2). +# If not set, Airflow uses a base template. +# +# Example: html_content_template = /path/to/my_html_content_template_file +# +# Variable: AIRFLOW__EMAIL__HTML_CONTENT_TEMPLATE +# +# html_content_template = + +# Email address that will be used as sender address. +# It can either be raw email or the complete address in a format ``Sender Name `` +# +# Example: from_email = Airflow +# +# Variable: AIRFLOW__EMAIL__FROM_EMAIL +# +from_email = $AIRFLOW__SMTP__SMTP_MAIL_FROM + +# ssl context to use when using SMTP and IMAP SSL connections. By default, the context is "default" +# which sets it to ``ssl.create_default_context()`` which provides the right balance between +# compatibility and security, it however requires that certificates in your operating system are +# updated and that SMTP/IMAP servers of yours have valid certificates that have corresponding public +# keys installed on your machines. You can switch it to "none" if you want to disable checking +# of the certificates, but it is not recommended as it allows MITM (man-in-the-middle) attacks +# if your infrastructure is not sufficiently secured. It should only be set temporarily while you +# are fixing your certificate configuration. This can be typically done by upgrading to newer +# version of the operating system you run Airflow components on,by upgrading/refreshing proper +# certificates in the OS or by updating certificates for your mail servers. +# +# Example: ssl_context = default +# +# Variable: AIRFLOW__EMAIL__SSL_CONTEXT +# +# ssl_context = default + +[smtp] +# If you want airflow to send emails on retries, failure, and you want to use +# the airflow.utils.email.send_email_smtp function, you have to configure an +# smtp server here + +# +# Variable: AIRFLOW__SMTP__SMTP_HOST +# +smtp_host = $AIRFLOW__SMTP__SMTP_HOST + +# +# Variable: AIRFLOW__SMTP__SMTP_STARTTLS +# +smtp_starttls = True + +# +# Variable: AIRFLOW__SMTP__SMTP_SSL +# +smtp_ssl = False + +# +# Example: smtp_user = airflow +# +# Variable: AIRFLOW__SMTP__SMTP_USER +# +smtp_user = $AIRFLOW__SMTP__SMTP_USER + +# +# Example: smtp_password = airflow +# +# Variable: AIRFLOW__SMTP__SMTP_PASSWORD +# +smtp_password = $AIRFLOW__SMTP__SMTP_PASSWORD + +# +# Variable: AIRFLOW__SMTP__SMTP_PORT +# +smtp_port = $AIRFLOW__SMTP__SMTP_PORT + +# +# Variable: AIRFLOW__SMTP__SMTP_MAIL_FROM +# +smtp_mail_from = $AIRFLOW__SMTP__SMTP_MAIL_FROM + +# +# Variable: AIRFLOW__SMTP__SMTP_TIMEOUT +# +smtp_timeout = 30 + +# +# Variable: AIRFLOW__SMTP__SMTP_RETRY_LIMIT +# +smtp_retry_limit = 5 + +[sentry] +# Sentry (https://docs.sentry.io) integration. Here you can supply +# additional configuration options based on the Python platform. See: +# https://docs.sentry.io/error-reporting/configuration/?platform=python. +# Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``, +# ``ignore_errors``, ``before_breadcrumb``, ``transport``. + +# Enable error reporting to Sentry +# +# Variable: AIRFLOW__SENTRY__SENTRY_ON +# +# sentry_on = false + +# +# Variable: AIRFLOW__SENTRY__SENTRY_DSN +# +# sentry_dsn = + +# Dotted path to a before_send function that the sentry SDK should be configured to use. +# +# Variable: AIRFLOW__SENTRY__BEFORE_SEND +# +# before_send = + +[scheduler] +# Task instances listen for external kill signal (when you clear tasks +# from the CLI or the UI), this defines the frequency at which they should +# listen (in seconds). +# +# Variable: AIRFLOW__SCHEDULER__JOB_HEARTBEAT_SEC +# +# job_heartbeat_sec = 5 + +# The scheduler constantly tries to trigger new tasks (look at the +# scheduler section in the docs for more information). This defines +# how often the scheduler should run (in seconds). +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC +# +# scheduler_heartbeat_sec = 5 + +# The frequency (in seconds) at which the LocalTaskJob should send heartbeat signals to the +# scheduler to notify it's still alive. If this value is set to 0, the heartbeat interval will default +# to the value of scheduler_zombie_task_threshold. +# +# Variable: AIRFLOW__SCHEDULER__LOCAL_TASK_JOB_HEARTBEAT_SEC +# +# local_task_job_heartbeat_sec = 0 + +# The number of times to try to schedule each DAG file +# -1 indicates unlimited number +# +# Variable: AIRFLOW__SCHEDULER__NUM_RUNS +# +# num_runs = -1 + +# Controls how long the scheduler will sleep between loops, but if there was nothing to do +# in the loop. i.e. if it scheduled something then it will start the next loop +# iteration straight away. +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_IDLE_SLEEP_TIME +# +# scheduler_idle_sleep_time = 1 + +# Number of seconds after which a DAG file is parsed. The DAG file is parsed every +# ``min_file_process_interval`` number of seconds. Updates to DAGs are reflected after +# this interval. Keeping this number low will increase CPU usage. +# +# Variable: AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL +# +# min_file_process_interval = 30 + +# How often (in seconds) to check for stale DAGs (DAGs which are no longer present in +# the expected files) which should be deactivated, as well as datasets that are no longer +# referenced and should be marked as orphaned. +# +# Variable: AIRFLOW__SCHEDULER__PARSING_CLEANUP_INTERVAL +# +# parsing_cleanup_interval = 60 + +# How long (in seconds) to wait after we have re-parsed a DAG file before deactivating stale +# DAGs (DAGs which are no longer present in the expected files). The reason why we need +# this threshold is to account for the time between when the file is parsed and when the +# DAG is loaded. The absolute maximum that this could take is `dag_file_processor_timeout`, +# but when you have a long timeout configured, it results in a significant delay in the +# deactivation of stale dags. +# +# Variable: AIRFLOW__SCHEDULER__STALE_DAG_THRESHOLD +# +# stale_dag_threshold = 50 + +# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. +# +# Variable: AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL +# +# dag_dir_list_interval = 300 + +# How often should stats be printed to the logs. Setting to 0 will disable printing stats +# +# Variable: AIRFLOW__SCHEDULER__PRINT_STATS_INTERVAL +# +# print_stats_interval = 30 + +# How often (in seconds) should pool usage stats be sent to StatsD (if statsd_on is enabled) +# +# Variable: AIRFLOW__SCHEDULER__POOL_METRICS_INTERVAL +# +# pool_metrics_interval = 5.0 + +# If the last scheduler heartbeat happened more than scheduler_health_check_threshold +# ago (in seconds), scheduler is considered unhealthy. +# This is used by the health check in the "/health" endpoint and in `airflow jobs check` CLI +# for SchedulerJob. +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_THRESHOLD +# +# scheduler_health_check_threshold = 30 + +# When you start a scheduler, airflow starts a tiny web server +# subprocess to serve a health check if this is set to True +# +# Variable: AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK +# +# enable_health_check = False + +# When you start a scheduler, airflow starts a tiny web server +# subprocess to serve a health check on this port +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_SERVER_PORT +# +# scheduler_health_check_server_port = 8974 + +# How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs +# +# Variable: AIRFLOW__SCHEDULER__ORPHANED_TASKS_CHECK_INTERVAL +# +# orphaned_tasks_check_interval = 300.0 + +# +# Variable: AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY +# +child_process_log_directory = $AIRFLOW_HOME/logs/scheduler + +# Local task jobs periodically heartbeat to the DB. If the job has +# not heartbeat in this many seconds, the scheduler will mark the +# associated task instance as failed and will re-schedule the task. +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULER_ZOMBIE_TASK_THRESHOLD +# +# scheduler_zombie_task_threshold = 300 + +# How often (in seconds) should the scheduler check for zombie tasks. +# +# Variable: AIRFLOW__SCHEDULER__ZOMBIE_DETECTION_INTERVAL +# +# zombie_detection_interval = 10.0 + +# Turn off scheduler catchup by setting this to ``False``. +# Default behavior is unchanged and +# Command Line Backfills still work, but the scheduler +# will not do scheduler catchup if this is ``False``, +# however it can be set on a per DAG basis in the +# DAG definition (catchup) +# +# Variable: AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT +# +# catchup_by_default = True + +# Setting this to True will make first task instance of a task +# ignore depends_on_past setting. A task instance will be considered +# as the first task instance of a task when there is no task instance +# in the DB with an execution_date earlier than it., i.e. no manual marking +# success will be needed for a newly added task to be scheduled. +# +# Variable: AIRFLOW__SCHEDULER__IGNORE_FIRST_DEPENDS_ON_PAST_BY_DEFAULT +# +# ignore_first_depends_on_past_by_default = True + +# This changes the batch size of queries in the scheduling main loop. +# This should not be greater than ``core.parallelism``. +# If this is too high, SQL query performance may be impacted by +# complexity of query predicate, and/or excessive locking. +# Additionally, you may hit the maximum allowable query length for your db. +# Set this to 0 to use the value of ``core.parallelism`` +# +# Variable: AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY +# +# max_tis_per_query = 16 + +# Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. +# If this is set to False then you should not run more than a single +# scheduler at once +# +# Variable: AIRFLOW__SCHEDULER__USE_ROW_LEVEL_LOCKING +# +# use_row_level_locking = True + +# Max number of DAGs to create DagRuns for per scheduler loop. +# +# Variable: AIRFLOW__SCHEDULER__MAX_DAGRUNS_TO_CREATE_PER_LOOP +# +# max_dagruns_to_create_per_loop = 10 + +# How many DagRuns should a scheduler examine (and lock) when scheduling +# and queuing tasks. +# +# Variable: AIRFLOW__SCHEDULER__MAX_DAGRUNS_PER_LOOP_TO_SCHEDULE +# +# max_dagruns_per_loop_to_schedule = 20 + +# Should the Task supervisor process perform a "mini scheduler" to attempt to schedule more tasks of the +# same DAG. Leaving this on will mean tasks in the same DAG execute quicker, but might starve out other +# dags in some circumstances +# +# Variable: AIRFLOW__SCHEDULER__SCHEDULE_AFTER_TASK_EXECUTION +# +# schedule_after_task_execution = True + +# The scheduler reads dag files to extract the airflow modules that are going to be used, +# and imports them ahead of time to avoid having to re-do it for each parsing process. +# This flag can be set to False to disable this behavior in case an airflow module needs to be freshly +# imported each time (at the cost of increased DAG parsing time). +# +# Variable: AIRFLOW__SCHEDULER__PARSING_PRE_IMPORT_MODULES +# +# parsing_pre_import_modules = True + +# The scheduler can run multiple processes in parallel to parse dags. +# This defines how many processes will run. +# +# Variable: AIRFLOW__SCHEDULER__PARSING_PROCESSES +# +# parsing_processes = 2 + +# One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``. +# The scheduler will list and sort the dag files to decide the parsing order. +# +# * ``modified_time``: Sort by modified time of the files. This is useful on large scale to parse the +# recently modified DAGs first. +# * ``random_seeded_by_host``: Sort randomly across multiple Schedulers but with same order on the +# same host. This is useful when running with Scheduler in HA mode where each scheduler can +# parse different DAG files. +# * ``alphabetical``: Sort by filename +# +# Variable: AIRFLOW__SCHEDULER__FILE_PARSING_SORT_MODE +# +# file_parsing_sort_mode = modified_time + +# Whether the dag processor is running as a standalone process or it is a subprocess of a scheduler +# job. +# +# Variable: AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR +# +# standalone_dag_processor = False + +# Only applicable if `[scheduler]standalone_dag_processor` is true and callbacks are stored +# in database. Contains maximum number of callbacks that are fetched during a single loop. +# +# Variable: AIRFLOW__SCHEDULER__MAX_CALLBACKS_PER_LOOP +# +# max_callbacks_per_loop = 20 + +# Only applicable if `[scheduler]standalone_dag_processor` is true. +# Time in seconds after which dags, which were not updated by Dag Processor are deactivated. +# +# Variable: AIRFLOW__SCHEDULER__DAG_STALE_NOT_SEEN_DURATION +# +# dag_stale_not_seen_duration = 600 + +# Turn off scheduler use of cron intervals by setting this to False. +# DAGs submitted manually in the web UI or with trigger_dag will still run. +# +# Variable: AIRFLOW__SCHEDULER__USE_JOB_SCHEDULE +# +# use_job_schedule = True + +# Allow externally triggered DagRuns for Execution Dates in the future +# Only has effect if schedule_interval is set to None in DAG +# +# Variable: AIRFLOW__SCHEDULER__ALLOW_TRIGGER_IN_FUTURE +# +# allow_trigger_in_future = False + +# How often to check for expired trigger requests that have not run yet. +# +# Variable: AIRFLOW__SCHEDULER__TRIGGER_TIMEOUT_CHECK_INTERVAL +# +# trigger_timeout_check_interval = 15 + +# Amount of time a task can be in the queued state before being retried or set to failed. +# +# Variable: AIRFLOW__SCHEDULER__TASK_QUEUED_TIMEOUT +# +# task_queued_timeout = 600.0 + +# How often to check for tasks that have been in the queued state for +# longer than `[scheduler] task_queued_timeout`. +# +# Variable: AIRFLOW__SCHEDULER__TASK_QUEUED_TIMEOUT_CHECK_INTERVAL +# +# task_queued_timeout_check_interval = 120.0 + +# The run_id pattern used to verify the validity of user input to the run_id parameter when +# triggering a DAG. This pattern cannot change the pattern used by scheduler to generate run_id +# for scheduled DAG runs or DAG runs triggered without changing the run_id parameter. +# +# Variable: AIRFLOW__SCHEDULER__ALLOWED_RUN_ID_PATTERN +# +# allowed_run_id_pattern = ^[A-Za-z0-9_.~:+-]+$ + +[triggerer] +# How many triggers a single Triggerer will run at once, by default. +# +# Variable: AIRFLOW__TRIGGERER__DEFAULT_CAPACITY +# +# default_capacity = 1000 + +# How often to heartbeat the Triggerer job to ensure it hasn't been killed. +# +# Variable: AIRFLOW__TRIGGERER__JOB_HEARTBEAT_SEC +# +# job_heartbeat_sec = 5 + +# If the last triggerer heartbeat happened more than triggerer_health_check_threshold +# ago (in seconds), triggerer is considered unhealthy. +# This is used by the health check in the "/health" endpoint and in `airflow jobs check` CLI +# for TriggererJob. +# +# Variable: AIRFLOW__TRIGGERER__TRIGGERER_HEALTH_CHECK_THRESHOLD +# +# triggerer_health_check_threshold = 30 + +[kerberos] +# +# Variable: AIRFLOW__KERBEROS__CCACHE +# +# ccache = /tmp/airflow_krb5_ccache + +# gets augmented with fqdn +# +# Variable: AIRFLOW__KERBEROS__PRINCIPAL +# +# principal = airflow + +# +# Variable: AIRFLOW__KERBEROS__REINIT_FREQUENCY +# +# reinit_frequency = 3600 + +# +# Variable: AIRFLOW__KERBEROS__KINIT_PATH +# +# kinit_path = kinit + +# +# Variable: AIRFLOW__KERBEROS__KEYTAB +# +# keytab = airflow.keytab + +# Allow to disable ticket forwardability. +# +# Variable: AIRFLOW__KERBEROS__FORWARDABLE +# +# forwardable = True + +# Allow to remove source IP from token, useful when using token behind NATted Docker host. +# +# Variable: AIRFLOW__KERBEROS__INCLUDE_IP +# +# include_ip = True + +[sensors] +# Sensor default timeout, 7 days by default (7 * 24 * 60 * 60). +# +# Variable: AIRFLOW__SENSORS__DEFAULT_TIMEOUT +# +# default_timeout = 604800 + +[imap] +# Options for IMAP provider. + +# ssl_context = diff --git a/containers/airflow/dags/brasil/sinan_ctl.py b/containers/airflow/dags/brasil/sinan_ctl.py deleted file mode 100644 index c60b57b0..00000000 --- a/containers/airflow/dags/brasil/sinan_ctl.py +++ /dev/null @@ -1,113 +0,0 @@ -import pendulum -import logging as logger - -from datetime import timedelta - -from airflow import DAG -from airflow.decorators import task -from epigraphhub.data.brasil.sinan import FTP_SINAN - -from epigraphhub.settings import env -from epigraphhub.connection import get_engine -from epigraphhub.data.brasil.sinan import ( - DISEASES, -) - -DEFAULT_ARGS = { - 'owner': 'epigraphhub', - 'depends_on_past': False, - 'email': ['epigraphhub@thegraphnetwork.org'], - 'email_on_failure': True, - 'email_on_retry': False, - 'retries': 2, - 'retry_delay': timedelta(minutes=2), -} - -engine = get_engine(credential_name=env.db.default_credential) - -with DAG( - dag_id='SINAN_UPDATE_CTL', - default_args=DEFAULT_ARGS, - tags=['SINAN', 'CTL'], - start_date=pendulum.datetime(2023, 4, 1, 12, 30), - catchup=False, - schedule='@daily', - description='A DAG to update the SINAN control table in EGH db', -): - - schema = 'brasil' - tablename ='sinan_update_ctl' - - @task(task_id='start') - def start_task(): - with engine.connect() as conn: - conn.execute( - f'CREATE TABLE IF NOT EXISTS {schema}.{tablename} (' - ' disease TEXT NOT NULL,' - ' year INT NOT NULL,' - ' path TEXT NOT NULL,' - ' prelim BOOL NOT NULL,' - ' to_final BOOL NOT NULL DEFAULT False,' - ' last_insert DATE' - ')' - ) - - @task(task_id='update_ctl_table') - def update_table(): - for disease in DISEASES.keys(): - dis = FTP_SINAN(disease) - - prelim_years = list(map(int, dis.get_years('prelim'))) - finals_years = list(map(int, dis.get_years('finais'))) - prelim_paths = dis.get_ftp_paths(prelim_years) - finals_paths = dis.get_ftp_paths(finals_years) - - for year, path in zip(prelim_years, prelim_paths): - if not year: continue - with engine.connect() as conn: - cur = conn.execute( - f'SELECT prelim FROM {schema}.{tablename}' - f" WHERE disease = '{disease}' AND year = {year}" - ) - prelim = cur.fetchone() - - if prelim is None: - with engine.connect() as conn: - conn.execute( - f'INSERT INTO {schema}.{tablename} (' - f'disease, year, path, prelim) VALUES (' - f"'{disease}', {year}, '{path}', True)" - ) - logger.debug(f'Insert {disease} {year} {path}') - - for year, path in zip(finals_years, finals_paths): - if not year: continue - with engine.connect() as conn: - cur = conn.execute( - f'SELECT prelim FROM {schema}.{tablename}' - f" WHERE disease = '{disease}' AND year = {year}" - ) - prelim = cur.fetchone() - - if prelim is None: - with engine.connect() as conn: - conn.execute( - f'INSERT INTO {schema}.{tablename} (' - f'disease, year, path, prelim) VALUES (' - f"'{disease}', {year}, '{path}', True)" - ) - logger.debug(f'Insert {disease} {year} {path}') - - elif prelim[0] is False: - with engine.connect() as conn: - conn.execute( - f'UPDATE {schema}.{tablename} SET' - f" prelim = False, to_final = True, path = '{path}'" - f" WHERE disease = '{disease}' AND year = {year}" - ) - logger.debug(f'Update to final {disease} {year} {path}') - - start = start_task() - update = update_table() - - start >> update diff --git a/containers/airflow/dags/brasil/sinan_drop_table.py b/containers/airflow/dags/brasil/sinan_drop_table.py deleted file mode 100644 index 55cf6d05..00000000 --- a/containers/airflow/dags/brasil/sinan_drop_table.py +++ /dev/null @@ -1,52 +0,0 @@ -import pendulum -import logging as logger - -from datetime import timedelta - -from airflow import DAG -from airflow.operators.python import PythonOperator - -from epigraphhub.settings import env -from epigraphhub.connection import get_engine -from epigraphhub.data.brasil.sinan import ( - normalize_str, -) - -DEFAULT_ARGS = { - 'owner': 'Admin', - 'depends_on_past': False, - 'email': ['epigraphhub@thegraphnetwork.org'], - 'email_on_failure': True, - 'email_on_retry': False, - 'retries': 2, - 'retry_delay': timedelta(minutes=2), -} - -engine = get_engine(credential_name=env.db.default_credential) - -with DAG( - dag_id='SINAN_DROP_TABLE', - default_args=DEFAULT_ARGS, - tags=['SINAN', 'CTL'], - start_date=pendulum.datetime(2023, 3, 27), - catchup=False, - schedule=None, #Only manually triggered - description='A DAG to delete a SINAN table in EGH db', -): - def drop_tables(disease: str): - dis = normalize_str(disease) - tablename = 'sinan_' + dis + '_m' - with engine.connect() as conn: - conn.execute( - f'DROP TABLE brasil.{tablename}' - ) - logger.warn(f'Dropped table {tablename} on schema brasil') - - delete_tables_task = PythonOperator( - task_id='drop_table', - python_callable=drop_tables, - op_kwargs={'disease': '{{ params.disease }}'}, - params={'disease': 'disease'}, - ) - - delete_tables_task diff --git a/containers/airflow/dags/colombia_dag.py b/containers/airflow/dags/colombia_dag.py index 70660263..3267425e 100644 --- a/containers/airflow/dags/colombia_dag.py +++ b/containers/airflow/dags/colombia_dag.py @@ -1,13 +1,10 @@ import pendulum import logging as logger from datetime import timedelta -from airflow.decorators import dag, task +from airflow import DAG +from airflow.decorators import task from airflow.operators.empty import EmptyOperator -from airflow.operators.python import BranchPythonOperator -from epigraphhub.data.colombia import ( - loading, - extract, -) +from airflow.operators.python import BranchExternalPythonOperator default_args = { @@ -21,14 +18,13 @@ "retry_delay": timedelta(minutes=1), } - -@dag( +with DAG( + dag_id='colombia', + tags=['COLOMBIA'], schedule="@daily", default_args=default_args, catchup=False, -) -def colombia(): - +): start = EmptyOperator( task_id="start", ) @@ -39,6 +35,8 @@ def colombia(): ) def compare(): + import logging as logger + from epigraphhub.data.colombia import extract if not extract.compare(): logger.info("Proceeding to update positive_cases_covid_d.") return "not_updated" @@ -52,13 +50,18 @@ def compare(): task_id="up_to_date", ) - check_dates = BranchPythonOperator( + check_dates = BranchExternalPythonOperator( task_id="check_last_update", python_callable=compare, + python='/opt/py310/bin/python3.10' ) - @task(task_id="load_into_db", retries=2) + @task.external_python( + task_id='load_into_db', python='/opt/py310/bin/python3.10' + ) def load(): + import logging as logger + from epigraphhub.data.colombia import loading loading.upload() logger.info("Table positive_cases_covid_d updated.") @@ -66,6 +69,3 @@ def load(): check_dates >> updated >> done check_dates >> outdated >> load() >> done - - -dag = colombia() diff --git a/containers/airflow/dags/owid_dag.py b/containers/airflow/dags/owid_dag.py index 011c8225..88348fd6 100644 --- a/containers/airflow/dags/owid_dag.py +++ b/containers/airflow/dags/owid_dag.py @@ -60,10 +60,10 @@ import pendulum import logging as logger from datetime import timedelta -from airflow.decorators import dag, task +from airflow import DAG +from airflow.decorators import task from airflow.operators.empty import EmptyOperator -from airflow.operators.python import BranchPythonOperator -from epigraphhub.data.owid import extract, transform, loading +from airflow.operators.python import BranchExternalPythonOperator default_args = { @@ -77,14 +77,13 @@ "retry_delay": timedelta(minutes=1), } - -@dag( +with DAG( + dag_id='owid', + tags=['OWID'], schedule="@daily", default_args=default_args, catchup=False, - tags=['OWID'] -) -def owid(): +): """ This method represents the DAG itself using the @dag decorator. The method has to be instantiated so the Scheduler can recognize as a DAG. OWID DAG @@ -137,12 +136,18 @@ def owid(): task_id="start", ) - @task(task_id="download_data") + @task.external_python( + task_id='download_data', python='/opt/py310/bin/python3.10' + ) def download_owid(): + import logging as logger + from epigraphhub.data.owid import extract extract.download() logger.info("OWID CSV downloaded") def comp_data(): + import logging as logger + from epigraphhub.data.owid import extract """ Returns: not_same_shape (str) : If evaluation is False, returns the string @@ -163,9 +168,10 @@ def comp_data(): logger.info("Table owid_covid up to date.") return "same_shape" - branch = BranchPythonOperator( + branch = BranchExternalPythonOperator( task_id="is_same_shape", python_callable=comp_data, + python='/opt/py310/bin/python3.10' ) not_same_shape = EmptyOperator( @@ -179,18 +185,30 @@ def comp_data(): trigger_rule="one_success", ) - @task(task_id="load_into_db") + @task.external_python( + task_id='load_into_db', python='/opt/py310/bin/python3.10' + ) def insert_into_db(): + import logging as logger + from epigraphhub.data.owid import loading loading.upload(remote=False) logger.info("Table owid_covid updated.") - @task(task_id="update_index") + @task.external_python( + task_id='update_index', python='/opt/py310/bin/python3.10' + ) def parse_index(): + import logging as logger + from epigraphhub.data.owid import transform transform.parse_indexes(remote=False) logger.info("location, iso_code and date indexes updated.") - @task(task_id="delete_csv") + @task.external_python( + task_id='delete_csv', python='/opt/py310/bin/python3.10' + ) def remove_csv(): + import logging as logger + from epigraphhub.data.owid import extract extract.remove() logger.info("OWID CSV removed.") @@ -209,6 +227,3 @@ def remove_csv(): branch >> not_same_shape >> insert_into_db() >> parse_index() >> done done >> remove_csv() - - -dag = owid() diff --git a/containers/airflow/dags/switzerland/foph_dag.py b/containers/airflow/dags/switzerland/foph_dag.py index e6c6e806..2ae34439 100644 --- a/containers/airflow/dags/switzerland/foph_dag.py +++ b/containers/airflow/dags/switzerland/foph_dag.py @@ -1,4 +1,4 @@ -""" +''' DISABLED @author Luã Bida Vacaro | github.com/luabida @date Last change on 2022-10-24 @@ -66,7 +66,7 @@ remove_csv_dir (PythonOperator) : This task will remove the FOPH CSV directory recursively. -""" + import pendulum import logging as logger @@ -309,4 +309,5 @@ def remove_csv_dir(): end >> clean -dag = FOPH() +dag = FOPH()i"" +''' diff --git a/containers/airflow/dags/switzerland/foph_metadata_dag.py b/containers/airflow/dags/switzerland/foph_metadata_dag.py index 33be9f75..d29a2abc 100644 --- a/containers/airflow/dags/switzerland/foph_metadata_dag.py +++ b/containers/airflow/dags/switzerland/foph_metadata_dag.py @@ -7,16 +7,12 @@ it will delete the table and re-insert it. """ import pendulum -import logging as logger -import pandas as pd from datetime import timedelta -from airflow.decorators import dag, task +from airflow import DAG +from airflow.decorators import task from airflow.operators.empty import EmptyOperator -from epigraphhub.data.foph import extract -from epigraphhub.connection import get_engine -from epigraphhub.settings import env default_args = { "owner": "epigraphhub", @@ -30,14 +26,13 @@ } -@dag( +with DAG( + dag_id='FOPH_METADATA', + tags = ['Metadata', 'CHE', 'FOPH', 'Switzerland'], schedule='@monthly', default_args=default_args, catchup=False, - tags = ['Metadata', 'CHE', 'FOPH', 'Switzerland'], - max_active_tasks=6 -) -def FOPH_METADATA(): +) as dag: start = EmptyOperator( task_id="start", ) @@ -47,74 +42,98 @@ def FOPH_METADATA(): trigger_rule="all_success", ) - tables_d = [ - [ - f'foph_{str(t).lower()}_d_meta', - str(u).split('/')[-1][:-4] - ] - for t, u - in extract.fetch() - ] # Daily tables - tables_default_w = [ - [ - f'foph_{str(t).lower()}_w_meta', - str(u).split('/')[-1][:-4] - ] - for t, u - in extract.fetch(freq='weekly') - ] # Weekly default tables - tables_by_age_w = [ - [ - f'foph_{str(t).lower()}_byage_w_meta', - str(u).split('/')[-1][:-4] - ] - for t, u - in extract.fetch(freq='weekly', by='age') - ] # Weekly by age tables - tables_by_sex_w = [ - [ - f'foph_{str(t).lower()}_bysex_w_meta', - str(u).split('/')[-1][:-4] - ] - for t, u - in extract.fetch(freq='weekly', by='sex') - ] # Weekly by sex tables - - tables = tables_d + tables_default_w + tables_by_age_w + tables_by_sex_w - - def extract_metadata(filename) -> pd.DataFrame: - # Creates the dataframe extracting the metadata from the API - df = extract.metadata(filename=filename) - properties = df[df.columns[0]]['properties'] - data = list() - for column, info in properties.items(): - try: - metadata = dict() - metadata['column_name'] = column - metadata['type'] = info['type'] - metadata['description'] = info['description'] - data.append(metadata) - except KeyError: - # Incompatible metadata column - continue - return pd.DataFrame(data) - - def load_to_db(tablename: str, dataframe: pd.DataFrame) -> None: - # Inserts metadata dataframe into DB - engine = get_engine(env.db.default_credential) - - with engine.connect() as conn: - dataframe.to_sql( - name=tablename, - con=conn, - schema='switzerland', - if_exists='replace' - ) - logger.info(f"{tablename} updated.") - - @task(task_id='load_metadata') + @task.external_python( + task_id='load_metadata', python='/opt/py310/bin/python3.10' + ) def load_metadata_tables(): - logger.info(f'Going to update: {[t[0] for t in tables]}') + import requests + import pandas as pd + import logging as logger + from epigraphhub.data.foph import extract + from epigraphhub.connection import get_engine + from epigraphhub.settings import env + + def fetch(freq: str = "daily", by: str = "default") -> tuple: + url = "https://www.covid19.admin.ch/api/data/context" + context = requests.get(url).json() + tables = context["sources"]["individual"]["csv"][freq] + if freq.lower() == "weekly": + if by.lower() == "age": + tables = tables["byAge"] + elif by.lower() == "sex": + tables = tables["bySex"] + else: + tables = tables[by] + for table, url in tables.items(): + yield table, url + + tables_d = [ + [ + f'foph_{str(t).lower()}_d_meta', + str(u).split('/')[-1][:-4] + ] + for t, u + in fetch() + ] # Daily tables + tables_default_w = [ + [ + f'foph_{str(t).lower()}_w_meta', + str(u).split('/')[-1][:-4] + ] + for t, u + in fetch(freq='weekly') + ] # Weekly default tables + tables_by_age_w = [ + [ + f'foph_{str(t).lower()}_byage_w_meta', + str(u).split('/')[-1][:-4] + ] + for t, u + in fetch(freq='weekly', by='age') + ] # Weekly by age tables + tables_by_sex_w = [ + [ + f'foph_{str(t).lower()}_bysex_w_meta', + str(u).split('/')[-1][:-4] + ] + for t, u + in fetch(freq='weekly', by='sex') + ] # Weekly by sex tables + + tables = tables_d + tables_default_w + tables_by_age_w + tables_by_sex_w + + def extract_metadata(filename) -> pd.DataFrame: + # Creates the dataframe extracting the metadata from the API + df = extract.metadata(filename=filename) + properties = df[df.columns[0]]['properties'] + data = list() + for column, info in properties.items(): + try: + metadata = dict() + metadata['column_name'] = column + metadata['type'] = info['type'] + metadata['description'] = info['description'] + data.append(metadata) + except KeyError: + # Incompatible metadata column + continue + return pd.DataFrame(data) + + def load_to_db(tablename: str, dataframe: pd.DataFrame) -> None: + # Inserts metadata dataframe into DB + engine = get_engine(env.db.default_credential) + + with engine.connect() as conn: + dataframe.to_sql( + name=tablename, + con=conn, + schema='switzerland', + if_exists='replace' + ) + + logger.info(f"{tablename} updated.") + logger.info(f'Going to update: {[t[0] for t in tables]}') + # Loops through all tables for table in tables: tablename, filename = table @@ -125,5 +144,3 @@ def load_metadata_tables(): load = load_metadata_tables() start >> load >> end - -dag = FOPH_METADATA() diff --git a/containers/airflow/dags/switzerland/foph_weekly_dag.py b/containers/airflow/dags/switzerland/foph_weekly_dag.py index 39a7f303..85a967cc 100644 --- a/containers/airflow/dags/switzerland/foph_weekly_dag.py +++ b/containers/airflow/dags/switzerland/foph_weekly_dag.py @@ -1,6 +1,6 @@ """ @author Luã Bida Vacaro | github.com/luabida -@date Last change on 2023-04-18 +@date Last change on 2023-10-05 NOTE: This DAG is a modified copy of foph DAG to fetch weekly data. FOPH stopped updating the daily dataset for Covid in 2023, @@ -8,18 +8,14 @@ took its place, keeping the same workflow with minor changes. """ import pendulum -import logging as logger +import requests from datetime import timedelta -from airflow.decorators import dag, task +from airflow import DAG +from airflow.decorators import task +from airflow.models import TaskInstance from airflow.operators.empty import EmptyOperator -from airflow.operators.python import PythonOperator, BranchPythonOperator - -from epigraphhub.data.foph import ( - extract, - loading, - transform, -) +from airflow.operators.python import ExternalPythonOperator, BranchExternalPythonOperator default_args = { @@ -34,14 +30,47 @@ } -@dag( +def fetch(freq: str = "daily", by: str = "default") -> tuple: + """ + A generator responsible for accessing FOPH and retrieve the CSV + relation, such as its Table name and URL as a tuple. + + Parameters + ---------- + freq : str + The frequency of the data (daily or weekly). + by : str + Available only for weekly data, fetches cases by age, + sex or default. + + Returns + ------- + table : str + Table name as in the json file. + url : str + URL to download the CSV. + """ + url = "https://www.covid19.admin.ch/api/data/context" + context = requests.get(url).json() + tables = context["sources"]["individual"]["csv"][freq] + if freq.lower() == "weekly": + if by.lower() == "age": + tables = tables["byAge"] + elif by.lower() == "sex": + tables = tables["bySex"] + else: + tables = tables[by] + for table, url in tables.items(): + yield table, url + + +with DAG( + dag_id='FOPH_WEEKLY', + tags = ['CHE', 'FOPH', 'Switzerland'], schedule='@weekly', default_args=default_args, catchup=False, - tags = ['CHE', 'FOPH', 'Switzerland'], - max_active_tasks=6 -) -def FOPH_WEEKLY(): +) as dag: start = EmptyOperator( task_id="start", ) @@ -51,17 +80,20 @@ def FOPH_WEEKLY(): trigger_rule="all_success", ) - tables_default = [[f'{t}_w', u] for t, u in extract.fetch(freq='weekly')] - tables_by_age = [[f'{t}_byage_w', u] for t, u in extract.fetch(freq='weekly', by='age')] - tables_by_sex = [[f'{t}_bysex_w', u] for t, u in extract.fetch(freq='weekly', by='sex')] - + tables_default = [[f'{t}_w', u] for t, u in fetch(freq='weekly')] + tables_by_age = [[f'{t}_byage_w', u] for t, u in fetch(freq='weekly', by='age')] + tables_by_sex = [[f'{t}_bysex_w', u] for t, u in fetch(freq='weekly', by='sex')] tables = tables_default + tables_by_age + tables_by_sex def download(url): + import logging as logger + from epigraphhub.data.foph import extract extract.download(url) logger.info(f"{str(url).split('/')[-1]} downloaded.") def compare(tablename, url): + import logging as logger + from epigraphhub.data.foph import loading filename = str(url).split("/")[-1] try: @@ -76,32 +108,42 @@ def compare(tablename, url): return f"{tablename}_up_to_date" def load_to_db(table, url): + import logging as logger + from epigraphhub.data.foph import loading filename = str(url).split("/")[-1] loading.upload(table, filename) logger.info(f"foph_{table} updated.") def parse_table(table): + import logging as logger + from epigraphhub.data.foph import transform transform.parse_date_region(table) logger.info(f"geoRegion and date index updated on foph_{table.lower()}") - @task(trigger_rule="all_done") + @task.external_python( + task_id='all_done', python='/opt/py310/bin/python3.10' + ) def remove_csv_dir(): + import logging as logger + from epigraphhub.data.foph import extract extract.remove(entire_dir=True) logger.info("CSVs directory removed.") for table in tables: tablename, url = table - down = PythonOperator( + down = ExternalPythonOperator( task_id=tablename, python_callable=download, op_kwargs={"url": url}, + python='/opt/py310/bin/python3.10' ) - comp = BranchPythonOperator( + comp = BranchExternalPythonOperator( task_id=f"comp_{tablename}", python_callable=compare, op_kwargs={"tablename": tablename, "url": url}, + python='/opt/py310/bin/python3.10' ) not_same_shape = EmptyOperator( @@ -111,16 +153,18 @@ def remove_csv_dir(): task_id=f"{tablename}_up_to_date", ) - load = PythonOperator( + load = ExternalPythonOperator( task_id=f"load_{tablename}", python_callable=load_to_db, op_kwargs={"table": tablename, "url": url}, + python='/opt/py310/bin/python3.10' ) - parse = PythonOperator( + parse = ExternalPythonOperator( task_id=f"parse_{tablename}", python_callable=parse_table, op_kwargs={"table": tablename}, + python='/opt/py310/bin/python3.10' ) done = EmptyOperator( @@ -145,5 +189,3 @@ def remove_csv_dir(): clean = remove_csv_dir() end >> clean - -dag = FOPH_WEEKLY() diff --git a/containers/airflow/scripts/webserver_config.py b/containers/airflow/envs/README.md similarity index 100% rename from containers/airflow/scripts/webserver_config.py rename to containers/airflow/envs/README.md diff --git a/containers/airflow/envs/epigraphhub.txt b/containers/airflow/envs/epigraphhub.txt new file mode 100644 index 00000000..eead9334 --- /dev/null +++ b/containers/airflow/envs/epigraphhub.txt @@ -0,0 +1 @@ +epigraphhub >= 2.1.0 diff --git a/containers/airflow/envs/pysus.txt b/containers/airflow/envs/pysus.txt new file mode 100644 index 00000000..79e71e0b --- /dev/null +++ b/containers/airflow/envs/pysus.txt @@ -0,0 +1 @@ +pysus >= 0.10.2 diff --git a/containers/airflow/scripts/entrypoint.sh b/containers/airflow/scripts/entrypoint.sh index f26d5992..7ccfd3ff 100755 --- a/containers/airflow/scripts/entrypoint.sh +++ b/containers/airflow/scripts/entrypoint.sh @@ -1,20 +1,337 @@ #!/usr/bin/env bash +# https://github.com/apache/airflow/blob/main/scripts/docker/entrypoint_prod.sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# Might be empty +AIRFLOW_COMMAND="${1:-}" -set -e +set -euo pipefail -# prepare the conda environment -is_conda_in_path=$(echo $PATH|grep -m 1 --count /opt/conda/) +# This one is to workaround https://github.com/apache/airflow/issues/17546 +# issue with /usr/lib/-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block +# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues +# by users of the prod image, we implement the workaround now. +# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any +# binary started and a little memory used for Heap allocated by initialization of libstdc++ +# This overhead is not happening for binaries that already link dynamically libstdc++ +LD_PRELOAD="/usr/lib/$(uname -m)-linux-gnu/libstdc++.so.6" +export LD_PRELOAD -if [ $is_conda_in_path == 0 ]; then - export PATH="/opt/conda/condabin:/opt/conda/bin:$PATH" - echo "[II] included conda to the PATH" +function run_check_with_retries { + local cmd + cmd="${1}" + local countdown + countdown="${CONNECTION_CHECK_MAX_COUNT}" + + while true + do + set +e + local last_check_result + local res + last_check_result=$(eval "${cmd} 2>&1") + res=$? + set -e + if [[ ${res} == 0 ]]; then + echo + break + else + echo -n "." + countdown=$((countdown-1)) + fi + if [[ ${countdown} == 0 ]]; then + echo + echo "ERROR! Maximum number of retries (${CONNECTION_CHECK_MAX_COUNT}) reached." + echo + echo "Last check result:" + echo "$ ${cmd}" + echo "${last_check_result}" + echo + exit 1 + else + sleep "${CONNECTION_CHECK_SLEEP_TIME}" + fi + done +} + +function run_nc() { + # Checks if it is possible to connect to the host using netcat. + # + # We want to avoid misleading messages and perform only forward lookup of the service IP address. + # Netcat when run without -n performs both forward and reverse lookup and fails if the reverse + # lookup name does not match the original name even if the host is reachable via IP. This happens + # randomly with docker-compose in GitHub Actions. + # Since we are not using reverse lookup elsewhere, we can perform forward lookup in python + # And use the IP in NC and add '-n' switch to disable any DNS use. + # Even if this message might be harmless, it might hide the real reason for the problem + # Which is the long time needed to start some services, seeing this message might be totally misleading + # when you try to analyse the problem, that's why it's best to avoid it, + local host="${1}" + local port="${2}" + local ip + ip=$(python -c "import socket; print(socket.gethostbyname('${host}'))") + nc -zvvn "${ip}" "${port}" +} + + +function wait_for_connection { + # Waits for Connection to the backend specified via URL passed as first parameter + # Detects backend type depending on the URL schema and assigns + # default port numbers if not specified in the URL. + # Then it loops until connection to the host/port specified can be established + # It tries `CONNECTION_CHECK_MAX_COUNT` times and sleeps `CONNECTION_CHECK_SLEEP_TIME` between checks + local connection_url + connection_url="${1}" + local detected_backend + detected_backend=$(python -c "from urllib.parse import urlsplit; import sys; print(urlsplit(sys.argv[1]).scheme)" "${connection_url}") + local detected_host + detected_host=$(python -c "from urllib.parse import urlsplit; import sys; print(urlsplit(sys.argv[1]).hostname or '')" "${connection_url}") + local detected_port + detected_port=$(python -c "from urllib.parse import urlsplit; import sys; print(urlsplit(sys.argv[1]).port or '')" "${connection_url}") + + echo BACKEND="${BACKEND:=${detected_backend}}" + readonly BACKEND + + if [[ -z "${detected_port=}" ]]; then + if [[ ${BACKEND} == "postgres"* ]]; then + detected_port=5432 + elif [[ ${BACKEND} == "mysql"* ]]; then + detected_port=3306 + elif [[ ${BACKEND} == "mssql"* ]]; then + detected_port=1433 + elif [[ ${BACKEND} == "redis"* ]]; then + detected_port=6379 + elif [[ ${BACKEND} == "amqp"* ]]; then + detected_port=5672 + fi + fi + + detected_host=${detected_host:="localhost"} + + # Allow the DB parameters to be overridden by environment variable + echo DB_HOST="${DB_HOST:=${detected_host}}" + readonly DB_HOST + + echo DB_PORT="${DB_PORT:=${detected_port}}" + readonly DB_PORT + if [[ -n "${DB_HOST=}" ]] && [[ -n "${DB_PORT=}" ]]; then + run_check_with_retries "run_nc ${DB_HOST@Q} ${DB_PORT@Q}" + else + >&2 echo "The connection details to the broker could not be determined. Connectivity checks were skipped." + fi +} + +function create_www_user() { + local local_password="" + # Warning: command environment variables (*_CMD) have priority over usual configuration variables + # for configuration parameters that require sensitive information. This is the case for the SQL database + # and the broker backend in this entrypoint script. + if [[ -n "${_AIRFLOW_WWW_USER_PASSWORD_CMD=}" ]]; then + local_password=$(eval "${_AIRFLOW_WWW_USER_PASSWORD_CMD}") + unset _AIRFLOW_WWW_USER_PASSWORD_CMD + elif [[ -n "${_AIRFLOW_WWW_USER_PASSWORD=}" ]]; then + local_password="${_AIRFLOW_WWW_USER_PASSWORD}" + unset _AIRFLOW_WWW_USER_PASSWORD + fi + if [[ -z ${local_password} ]]; then + echo + echo "ERROR! Airflow Admin password not set via _AIRFLOW_WWW_USER_PASSWORD or _AIRFLOW_WWW_USER_PASSWORD_CMD variables!" + echo + exit 1 + fi + + airflow users create \ + --username "${_AIRFLOW_WWW_USER_USERNAME="admin"}" \ + --firstname "${_AIRFLOW_WWW_USER_FIRSTNAME="Airflow"}" \ + --lastname "${_AIRFLOW_WWW_USER_LASTNAME="Admin"}" \ + --email "${_AIRFLOW_WWW_USER_EMAIL="airflowadmin@example.com"}" \ + --role "${_AIRFLOW_WWW_USER_ROLE="Admin"}" \ + --password "${local_password}" || true +} + +function create_system_user_if_missing() { + # This is needed in case of OpenShift-compatible container execution. In case of OpenShift random + # User id is used when starting the image, however group 0 is kept as the user group. Our production + # Image is OpenShift compatible, so all permissions on all folders are set so that 0 group can exercise + # the same privileges as the default "airflow" user, this code checks if the user is already + # present in /etc/passwd and will create the system user dynamically, including setting its + # HOME directory to the /home/airflow so that (for example) the ${HOME}/.local folder where airflow is + # Installed can be automatically added to PYTHONPATH + if ! whoami &> /dev/null; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${AIRFLOW_USER_HOME_DIR}:/sbin/nologin" \ + >> /etc/passwd + fi + export HOME="${AIRFLOW_USER_HOME_DIR}" + fi +} + +function set_pythonpath_for_root_user() { + # Airflow is installed as a local user application which means that if the container is running as root + # the application is not available. because Python then only load system-wide applications. + # Now also adds applications installed as local user "airflow". + if [[ $UID == "0" ]]; then + local python_major_minor + python_major_minor="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)" + export PYTHONPATH="${AIRFLOW_USER_HOME_DIR}/.local/lib/python${python_major_minor}/site-packages:${PYTHONPATH:-}" + >&2 echo "The container is run as root user. For security, consider using a regular user account." + fi +} + +function wait_for_airflow_db() { + # Wait for the command to run successfully to validate the database connection. + run_check_with_retries "airflow db check" +} + +function migrate_db() { + # Runs airflow db migrate + airflow db migrate || true +} + +function wait_for_celery_broker() { + # Verifies connection to Celery Broker + local executor + executor="$(airflow config get-value core executor)" + if [[ "${executor}" == "CeleryExecutor" ]]; then + local connection_url + connection_url="$(airflow config get-value celery broker_url)" + wait_for_connection "${connection_url}" + fi +} + +function exec_to_bash_or_python_command_if_specified() { + # If one of the commands: 'bash', 'python' is used, either run appropriate + # command with exec + if [[ ${AIRFLOW_COMMAND} == "bash" ]]; then + shift + exec "/bin/bash" "${@}" + elif [[ ${AIRFLOW_COMMAND} == "python" ]]; then + shift + exec "python" "${@}" + fi +} + +function check_uid_gid() { + if [[ $(id -g) == "0" ]]; then + return + fi + if [[ $(id -u) == "50000" ]]; then + >&2 echo + >&2 echo "WARNING! You should run the image with GID (Group ID) set to 0" + >&2 echo " even if you use 'airflow' user (UID=50000)" + >&2 echo + >&2 echo " You started the image with UID=$(id -u) and GID=$(id -g)" + >&2 echo + >&2 echo " This is to make sure you can run the image with an arbitrary UID in the future." + >&2 echo + >&2 echo " See more about it in the Airflow's docker image documentation" + >&2 echo " http://airflow.apache.org/docs/docker-stack/entrypoint" + >&2 echo + # We still allow the image to run with `airflow` user. + return + else + >&2 echo + >&2 echo "ERROR! You should run the image with GID=0" + >&2 echo + >&2 echo " You started the image with UID=$(id -u) and GID=$(id -g)" + >&2 echo + >&2 echo "The image should always be run with GID (Group ID) set to 0 regardless of the UID used." + >&2 echo " This is to make sure you can run the image with an arbitrary UID." + >&2 echo + >&2 echo " See more about it in the Airflow's docker image documentation" + >&2 echo " http://airflow.apache.org/docs/docker-stack/entrypoint" + # This will not work so we fail hard + exit 1 + fi +} + +# In Airflow image we are setting PIP_USER variable to true, in order to install all the packages +# by default with the ``--user`` flag. However this is a problem if a virtualenv is created later +# which happens in PythonVirtualenvOperator. We are unsetting this variable here, so that it is +# not set when PIP is run by Airflow later on +unset PIP_USER + +check_uid_gid + +# Set umask to 0002 to make all the directories created by the current user group-writeable +# This allows the same directories to be writeable for any arbitrary user the image will be +# run with, when the directory is created on a mounted volume and when that volume is later +# reused with a different UID (but with GID=0) +umask 0002 + +CONNECTION_CHECK_MAX_COUNT=${CONNECTION_CHECK_MAX_COUNT:=20} +readonly CONNECTION_CHECK_MAX_COUNT + +CONNECTION_CHECK_SLEEP_TIME=${CONNECTION_CHECK_SLEEP_TIME:=3} +readonly CONNECTION_CHECK_SLEEP_TIME + +create_system_user_if_missing +set_pythonpath_for_root_user +if [[ "${CONNECTION_CHECK_MAX_COUNT}" -gt "0" ]]; then + wait_for_airflow_db +fi + +if [[ -n "${_AIRFLOW_DB_UPGRADE=}" ]] || [[ -n "${_AIRFLOW_DB_MIGRATE=}" ]] ; then + migrate_db +fi + +if [[ -n "${_AIRFLOW_DB_UPGRADE=}" ]] ; then + >&2 echo "WARNING: Environment variable '_AIRFLOW_DB_UPGRADE' is deprecated please use '_AIRFLOW_DB_MIGRATE' instead" fi -echo "[II] activate epigraphhub" -source activate epigraphhub +if [[ -n "${_AIRFLOW_WWW_USER_CREATE=}" ]] ; then + create_www_user +fi -if [ $# -ne 0 ] - then - echo "Running: ${@}" - $(${@}) +if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then + >&2 echo + >&2 echo "!!!!! Installing additional requirements: '${_PIP_ADDITIONAL_REQUIREMENTS}' !!!!!!!!!!!!" + >&2 echo + >&2 echo "WARNING: This is a development/test feature only. NEVER use it in production!" + >&2 echo " Instead, build a custom image as described in" + >&2 echo + >&2 echo " https://airflow.apache.org/docs/docker-stack/build.html" + >&2 echo + >&2 echo " Adding requirements at container startup is fragile and is done every time" + >&2 echo " the container starts, so it is only useful for testing and trying out" + >&2 echo " of adding dependencies." + >&2 echo + pip install --root-user-action ignore --no-cache-dir ${_PIP_ADDITIONAL_REQUIREMENTS} fi + + +# The `bash` and `python` commands should also verify the basic connections +# So they are run after the DB check +exec_to_bash_or_python_command_if_specified "${@}" + +# Remove "airflow" if it is specified as airflow command +# This way both command types work the same way: +# +# docker run IMAGE airflow webserver +# docker run IMAGE webserver +# +if [[ ${AIRFLOW_COMMAND} == "airflow" ]]; then + AIRFLOW_COMMAND="${2:-}" + shift +fi + +# Note: the broker backend configuration concerns only a subset of Airflow components +if [[ ${AIRFLOW_COMMAND} =~ ^(scheduler|celery)$ ]] \ + && [[ "${CONNECTION_CHECK_MAX_COUNT}" -gt "0" ]]; then + wait_for_celery_broker +fi + +exec "airflow" "${@}" diff --git a/containers/airflow/scripts/init-db.sh b/containers/airflow/scripts/init-db.sh deleted file mode 100755 index 5d018276..00000000 --- a/containers/airflow/scripts/init-db.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# source: https://airflow.apache.org/docs/apache-airflow/stable/docker-compose.yaml - -if [[ -z "${AIRFLOW_UID}" ]]; then - echo - echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" - echo "If you are on Linux, you SHOULD follow the instructions below to set " - echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." - echo "For other operating systems you can get rid of the warning with manually created .env file:" - echo " See: https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#setting-the-right-airflow-user" - echo -fi -one_meg=1048576 -mem_available=$(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / one_meg)) -cpus_available=$(grep -cE 'cpu[0-9]+' /proc/stat) -disk_available=$(df / | tail -1 | awk '{print $4}') -warning_resources="false" -if (( mem_available < 4000 )) ; then - echo - echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m" - echo "At least 4GB of memory required. You have $(numfmt --to iec $((mem_available * one_meg)))" - echo - warning_resources="true" -fi -if (( cpus_available < 2 )); then - echo - echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m" - echo "At least 2 CPUs recommended. You have ${cpus_available}" - echo - warning_resources="true" -fi -if (( disk_available < one_meg * 10 )); then - echo - echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m" - echo "At least 10 GBs recommended. You have $(numfmt --to iec $((disk_available * 1024 )))" - echo - warning_resources="true" -fi -if [[ ${warning_resources} == "true" ]]; then - echo - echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" - echo "Please follow the instructions to increase amount of resources available:" - echo " https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#before-you-begin" - echo -fi -mkdir -p /sources/logs /sources/dags /sources/plugins - -airflow db init diff --git a/containers/airflow/scripts/startup.sh b/containers/airflow/scripts/startup.sh index 3602a3e4..d58e26e7 100755 --- a/containers/airflow/scripts/startup.sh +++ b/containers/airflow/scripts/startup.sh @@ -4,7 +4,7 @@ set -e # initdb echo "=========== init-db ===========" -. /opt/scripts/init-db.sh +airflow db init # create admin user echo "=========== init-db ===========" diff --git a/containers/compose-airflow.yaml b/containers/compose-airflow.yaml new file mode 100644 index 00000000..7f7fbd5d --- /dev/null +++ b/containers/compose-airflow.yaml @@ -0,0 +1,243 @@ +version: '3.8' +x-airflow-common: + &airflow-common + build: + context: .. + dockerfile: containers/airflow/Dockerfile + args: + POSTGRES_EPIGRAPH_HOST: ${POSTGRES_EPIGRAPH_HOST} + POSTGRES_EPIGRAPH_PORT: ${POSTGRES_EPIGRAPH_PORT} + POSTGRES_EPIGRAPH_USER: ${POSTGRES_EPIGRAPH_USER} + POSTGRES_EPIGRAPH_PASSWORD: ${POSTGRES_EPIGRAPH_PASSWORD} + POSTGRES_EPIGRAPH_DB: ${POSTGRES_EPIGRAPH_DB} + environment: + &airflow-common-env + AIRFLOW_HOME: /opt/airflow + AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY} + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' + AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' + volumes: + - ${AIRFLOW_PROJ_DIR}/dags:/opt/airflow/dags + - ${AIRFLOW_PROJ_DIR}/logs:/opt/airflow/logs + - ${AIRFLOW_PROJ_DIR}/config:/opt/airflow/config + - ${AIRFLOW_PROJ_DIR}/plugins:/opt/airflow/plugins + user: "${AIRFLOW_UID}:0" + depends_on: + &airflow-common-depends-on + redis: + condition: service_healthy + postgres: + condition: service_healthy + +services: + postgres: + image: postgres:13 + environment: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow + volumes: + - postgres-db-volume:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "airflow"] + interval: 10s + retries: 5 + start_period: 5s + restart: always + + redis: + image: redis:latest + expose: + - 6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 30s + retries: 50 + start_period: 30s + restart: always + + webserver: + <<: *airflow-common + command: webserver + ports: + - "8080:${AIRFLOW_PORT}" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully + + scheduler: + <<: *airflow-common + command: scheduler + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8974/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully + + worker: + <<: *airflow-common + command: celery worker + healthcheck: + # yamllint disable rule:line-length + test: + - "CMD-SHELL" + - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + environment: + <<: *airflow-common-env + # Required to handle warm shutdown of the celery workers properly + # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation + DUMB_INIT_SETSID: "0" + restart: always + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully + + triggerer: + <<: *airflow-common + command: triggerer + healthcheck: + test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully + + startup: + <<: *airflow-common + entrypoint: /bin/bash + # yamllint disable rule:line-length + command: + - -c + - | + function ver() { + printf "%04d%04d%04d%04d" $${1//./ } + } + airflow_version=$$(AIRFLOW__LOGGING__LOGGING_LEVEL=INFO && gosu airflow airflow version) + airflow_version_comparable=$$(ver $${airflow_version}) + min_airflow_version=2.2.0 + min_airflow_version_comparable=$$(ver $${min_airflow_version}) + if (( airflow_version_comparable < min_airflow_version_comparable )); then + echo + echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m" + echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!" + echo + exit 1 + fi + if [[ -z "${AIRFLOW_UID}" ]]; then + echo + echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" + echo "If you are on Linux, you SHOULD follow the instructions below to set " + echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." + echo "For other operating systems you can get rid of the warning with manually created .env file:" + echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" + echo + fi + one_meg=1048576 + mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg)) + cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat) + disk_available=$$(df / | tail -1 | awk '{print $$4}') + warning_resources="false" + if (( mem_available < 4000 )) ; then + echo + echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m" + echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))" + echo + warning_resources="true" + fi + if (( cpus_available < 2 )); then + echo + echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m" + echo "At least 2 CPUs recommended. You have $${cpus_available}" + echo + warning_resources="true" + fi + if (( disk_available < one_meg * 10 )); then + echo + echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m" + echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))" + echo + warning_resources="true" + fi + if [[ $${warning_resources} == "true" ]]; then + echo + echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" + echo "Please follow the instructions to increase amount of resources available:" + echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" + echo + fi + mkdir -p /sources/logs /sources/dags /sources/plugins /sources/config + chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins,config} + exec /entrypoint airflow version + # yamllint enable rule:line-length + environment: + <<: *airflow-common-env + _AIRFLOW_DB_MIGRATE: 'true' + _AIRFLOW_WWW_USER_CREATE: 'true' + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD} + user: "0:0" + volumes: + - ${AIRFLOW_PROJ_DIR}:/sources + + airflow-cli: + <<: *airflow-common + profiles: + - debug + environment: + <<: *airflow-common-env + CONNECTION_CHECK_MAX_COUNT: "0" + # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252 + command: + - bash + - -c + - airflow + + # You can enable flower by adding "--profile flower" option e.g. docker-compose --profile flower up + # or by explicitly targeted on the command line e.g. docker-compose up flower. + # See: https://docs.docker.com/compose/profiles/ + flower: + <<: *airflow-common + command: celery flower + profiles: + - flower + ports: + - "5555:5555" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:5555/"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully + +volumes: + postgres-db-volume: