move standard, alibaba and common.sql provider to the new structure (a…

…pache#45964)

This one also adds some missing steps in provider migration script.
improves the output and instructions and make it more robust for
various variations of provider features and usage of imports.

With those changes `standard` provider was 100% moved automatically.

Also common.sql needs to be added in order to migrate standard in
order to build production image.

Alibaba was added accidentally during demo at dev call so we are
committing it as well since it is already migrated.

.github/actions/install-pre-commit/action.yml

@@ -24,10 +24,10 @@ inputs:
     default: "3.9"
   uv-version:
     description: 'uv version to use'
-    default: "0.5.20"  # Keep this comment to allow automatic replacement of uv version
+    default: "0.5.23"  # Keep this comment to allow automatic replacement of uv version
   pre-commit-version:
     description: 'pre-commit version to use'
-    default: "4.0.1"  # Keep this comment to allow automatic replacement of pre-commit version
+    default: "4.1.0"  # Keep this comment to allow automatic replacement of pre-commit version
   pre-commit-uv-version:
     description: 'pre-commit-uv version to use'
     default: "4.1.4"  # Keep this comment to allow automatic replacement of pre-commit-uv version

.github/boring-cyborg.yml

@@ -22,10 +22,7 @@ labelPRBasedOnFilePath:
     - providers/airbyte/**
 
   provider:alibaba:
-    - providers/src/airflow/providers/alibaba/**/*
-    - docs/apache-airflow-providers-alibaba/**/*
-    - providers/tests/alibaba/**/*
-    - providers/tests/system/alibaba/**/*
+    - providers/alibaba/**
 
   provider:amazon:
     - providers/src/airflow/providers/amazon/**/*
@@ -173,15 +170,10 @@ labelPRBasedOnFilePath:
     - providers/tests/system/common/io/**/*
 
   provider:common-sql:
-    - providers/src/airflow/providers/common/sql/**/*
-    - docs/apache-airflow-providers-common-sql/**/*
-    - providers/tests/common/sql/**/*
-    - providers/tests/system/common/sql/**/*
+    - providers/common/sql/**
 
   provider:standard:
-    - providers/src/airflow/providers/standard/**/*
-    - docs/apache-airflow-providers-standard/**/*
-    - providers/tests/standard/**/*
+    - providers/standard/**
 
   provider:databricks:
     - providers/src/airflow/providers/databricks/**/*
@@ -647,12 +639,12 @@ labelPRBasedOnFilePath:
     - airflow/cli/commands/local_commands/triggerer_command.py
     - airflow/jobs/triggerer_job_runner.py
     - airflow/models/trigger.py
-    - providers/src/airflow/providers/standard/triggers/**/*
+    - providers/standard/src/airflow/providers/standard/triggers/**/*
     - tests/cli/commands/local_commands/test_triggerer_command.py
     - tests/jobs/test_triggerer_job.py
     - tests/models/test_trigger.py
     - tests/jobs/test_triggerer_job_logging.py
-    - providers/tests/standard/triggers/**/*
+    - providers/standard/tests/provider_tests/standard/triggers/**/*
 
   area:Serialization:
     - airflow/serialization/**/*

.github/workflows/test-provider-packages.yml

@@ -109,6 +109,10 @@ jobs:
         run: >
           breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
           --package-format ${{ matrix.package-format }}
+      - name: "Prepare task-sdk package: ${{ matrix.package-format }}"
+        run: >
+          breeze release-management prepare-task-sdk-package --version-suffix-for-pypi dev0
+          --package-format ${{ matrix.package-format }}
       - name: "Verify ${{ matrix.package-format }} packages with twine"
         run: |
           uv tool uninstall twine || true

.pre-commit-config.yaml

@@ -180,14 +180,6 @@ repos:
         additional_dependencies: ['rich>=12.4.4', "ruff==0.8.1"]
         files: ^providers/src/airflow/providers/.*\.py$
         require_serial: true
-      - id: update-common-sql-api-stubs
-        name: Check and update common.sql API stubs
-        entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py
-        language: python
-        files: ^scripts/ci/pre_commit/update_common_sql_api\.py|^providers/src/airflow/providers/common/sql/.*\.pyi?$
-        additional_dependencies: ['rich>=12.4.4', 'mypy==1.9.0', 'black==24.10.0', 'jinja2']
-        pass_filenames: false
-        require_serial: true
       - id: update-black-version
         name: Update black versions everywhere (manual)
         entry: ./scripts/ci/pre_commit/update_black_version.py
@@ -475,9 +467,7 @@ repos:
         language: python
         files: |
           (?x)
-          ^providers/src/airflow/providers/.*\.py$ |
-          ^providers/src/airflow/providers/.*/provider\.yaml$ |
-          ^providers/tests/.*\.py$ |
+          ^providers/.*/provider\.yaml$|
           ^scripts/ci/pre_commit/update_providers_dependencies\.py$
         pass_filenames: false
         additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml', 'tomli']
@@ -517,6 +507,14 @@ repos:
         files: ^docs/.*/diagram_[^/]*\.py$
         pass_filenames: true
         additional_dependencies: ['rich>=12.4.4', "diagrams>=0.23.4"]
+      - id: generate-volumes-for-sources
+        name: Generate volumes for docker compose
+        entry: ./scripts/ci/pre_commit/generate_volumes_for_sources.py
+        language: python
+        files: ^providers/.*/provider.yaml$
+        pass_filenames: false
+        require_serial: true
+        additional_dependencies: ['rich>=12.4.4']
       - id: update-supported-versions
         name: Updates supported versions in documentation
         entry: ./scripts/ci/pre_commit/supported_versions.py
@@ -623,6 +621,8 @@ repos:
           ^airflow/config_templates/|
           ^airflow/models/baseoperator.py$|
           ^airflow/operators/__init__.py$|
+          ^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute.py$|
+          ^providers/src/airflow/providers/fab/www/static/css/bootstrap-theme.css$|
           ^providers/src/airflow/providers/amazon/aws/hooks/emr.py$|
           ^providers/src/airflow/providers/amazon/aws/operators/emr.py$|
           ^providers/src/airflow/providers/apache/cassandra/hooks/cassandra.py$|
@@ -672,8 +672,11 @@ repos:
           ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint.sh$|
           ^tests/|
           ^providers/tests/|
+          ^providers/.*/tests/|
           ^.pre-commit-config\.yaml$|
           ^.*CHANGELOG\.(rst|txt)$|
+          ^.*changelog\.(rst|txt)$|
+          ^.*commits\.(rst|txt)$|
           ^.*RELEASE_NOTES\.rst$|
           ^contributing-docs/03_contributors_quick_start.rst$|
           ^.*\.(png|gif|jp[e]?g|tgz|lock)$|
@@ -724,6 +727,8 @@ repos:
           ^airflow/operators/.*$|
           ^providers/src/airflow/providers/.*$|
           ^providers/src/airflow/providers/standard/sensors/.*$|
+          ^providers/.*/src/airflow/providers/.*$|
+          ^providers/.*/src/airflow/providers/standard/sensors/.*$|
           ^dev/provider_packages/.*$
       - id: check-base-operator-usage
         language: pygrep
@@ -733,8 +738,9 @@ repos:
         pass_filenames: true
         files: >
           (?x)
-          ^providers/src/airflow/providers/.*\.py$
-        exclude: providers/src/airflow/providers/standard/operators/bash.py|providers/src/airflow/providers/standard/operators/python.py|providers/src/airflow/providers/standard/sensors/external_task.py
+          ^providers/src/airflow/providers/.*\.py$|
+          ^providers/.*/src/airflow/providers/.*\.py$
+        exclude: providers/standard/.*/.*\.py$
       - id: check-get-lineage-collector-providers
         language: python
         name: Check providers import hook lineage code from compat
@@ -1205,10 +1211,12 @@ repos:
           ^airflow/utils/file.py$ |
           ^airflow/utils/helpers.py$ |
           ^airflow/utils/log/secrets_masker.py$ |
-          ^providers/src/airflow/providers/ |
-          ^(providers/)?tests/ |
-          task_sdk/src/airflow/sdk/definitions/dag.py$ |
-          task_sdk/src/airflow/sdk/definitions/_internal/node.py$ |
+          ^providers/ |
+          ^tests/ |
+          ^providers/tests/ |
+          ^providers/.*/tests/ |
+          ^task_sdk/src/airflow/sdk/definitions/dag.py$ |
+          ^task_sdk/src/airflow/sdk/definitions/_internal/node.py$ |
           ^dev/.*\.py$ |
           ^scripts/.*\.py$ |
           ^docker_tests/.*$ |
@@ -1343,7 +1351,7 @@ repos:
         name: Run mypy for providers
         language: python
         entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages
-        files: ^providers/src/airflow/providers/.*\.py$|^providers/tests//.*\.py$|^providers/.*/src/.*\.py$|^providers/.*/tests/.*\.py$
+        files: ^providers/.*\.py$
         require_serial: true
         additional_dependencies: ['rich>=12.4.4']
       - id: mypy-providers
@@ -1392,7 +1400,7 @@ repos:
         name: Validate provider.yaml files
         entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py
         language: python
-        files: ^providers/src/airflow/providers/.*/provider\.yaml$|^providers/.*/src/provider\.yaml$
+        files: ^providers/.*/provider\.yaml$
         additional_dependencies: ['rich>=12.4.4']
         require_serial: true
       - id: check-template-fields-valid

Dockerfile

@@ -55,7 +55,7 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
 # Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
 ARG AIRFLOW_PIP_VERSION=24.3.1
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
-ARG AIRFLOW_UV_VERSION=0.5.20
+ARG AIRFLOW_UV_VERSION=0.5.23
 ARG AIRFLOW_USE_UV="false"
 ARG UV_HTTP_TIMEOUT="300"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"

Dockerfile.ci

@@ -915,13 +915,6 @@ function determine_airflow_to_use() {
         # Some packages might leave legacy typing module which causes test issues
         # shellcheck disable=SC2086
         ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} typing || true
-        if [[ ${LINK_PROVIDERS_TO_AIRFLOW_PACKAGE=} == "true" ]]; then
-            echo
-            echo "${COLOR_BLUE}Linking providers to airflow package as we are using them from mounted sources.${COLOR_RESET}"
-            echo
-            rm -rf /usr/local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow/providers
-            ln -s "${AIRFLOW_SOURCES}/providers/src/airflow/providers" "/usr/local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow/providers"
-        fi
     fi
 
     if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then
@@ -1268,9 +1261,9 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe
 # Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
 ARG AIRFLOW_PIP_VERSION=24.3.1
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
-ARG AIRFLOW_UV_VERSION=0.5.20
+ARG AIRFLOW_UV_VERSION=0.5.23
 # TODO(potiuk): automate with upgrade check (possibly)
-ARG AIRFLOW_PRE_COMMIT_VERSION="4.0.1"
+ARG AIRFLOW_PRE_COMMIT_VERSION="4.1.0"
 ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4"
 
 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \

airflow/utils/dag_parsing_context.py

@@ -19,6 +19,7 @@
 
 import warnings
 
+from airflow.sdk.definitions._internal.dag_parsing_context import _airflow_parsing_context_manager
 from airflow.sdk.definitions.context import get_parsing_context
 
 # TODO: Remove this module in Airflow 3.2
@@ -30,4 +31,4 @@
     stacklevel=2,
 )
 
-__all__ = ["get_parsing_context"]
+__all__ = ["get_parsing_context", "_airflow_parsing_context_manager"]

contributing-docs/08_static_code_checks.rst

@@ -280,6 +280,8 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | generate-pypi-readme                                      | Generate PyPI README                                   |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
+| generate-volumes-for-sources                              | Generate volumes for docker compose                    |         |
++-----------------------------------------------------------+--------------------------------------------------------+---------+
 | identity                                                  | Print checked files                                    |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | insert-license                                            | * Add license for all SQL files                        |         |
@@ -364,8 +366,6 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | update-chart-dependencies                                 | Update chart dependencies to latest (manual)           |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
-| update-common-sql-api-stubs                               | Check and update common.sql API stubs                  |         |
-+-----------------------------------------------------------+--------------------------------------------------------+---------+
 | update-er-diagram                                         | Update ER diagram                                      | *       |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | update-extras                                             | Update extras in documentation                         |         |

dev/breeze/doc/ci/02_images.md

@@ -443,8 +443,8 @@ can be used for CI images:
 | `ADDITIONAL_DEV_APT_DEPS`       |                            | Additional apt dev dependencies installed in the first part of the image                                          |
 | `ADDITIONAL_DEV_APT_ENV`        |                            | Additional env variables defined when installing dev deps                                                         |
 | `AIRFLOW_PIP_VERSION`           | `24.3.1`                   | `pip` version used.                                                                                               |
-| `AIRFLOW_UV_VERSION`            | `0.5.20`                   | `uv` version used.                                                                                                |
-| `AIRFLOW_PRE_COMMIT_VERSION`    | `4.0.1`                    | `pre-commit` version used.                                                                                        |
+| `AIRFLOW_UV_VERSION`            | `0.5.23`                   | `uv` version used.                                                                                                |
+| `AIRFLOW_PRE_COMMIT_VERSION`    | `4.1.0`                    | `pre-commit` version used.                                                                                        |
 | `AIRFLOW_PRE_COMMIT_UV_VERSION` | `4.1.4`                    | `pre-commit-uv` version used.                                                                                     |
 | `AIRFLOW_USE_UV`                | `true`                     | Whether to use UV for installation.                                                                               |
 | `PIP_PROGRESS_BAR`              | `on`                       | Progress bar for PIP installation                                                                                 |