diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index 88cab0a3b2523..3213bf8b3c2d1 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -40,7 +40,6 @@ body: - apache-pig - apache-pinot - apache-spark - - apache-sqoop - apprise - arangodb - asana @@ -89,7 +88,6 @@ body: - papermill - pgvector - pinecone - - plexus - postgres - presto - redis diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e5b5500f40dc6..818ca6120db93 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -709,18 +709,18 @@ This is the full list of those extras: .. START EXTRAS HERE aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, -apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, -apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud, +apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.webhdfs, apprise, +arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, +cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, fab, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie, oracle, otel, pagerduty, -pandas, papermill, password, pgvector, pinecone, pinot, plexus, postgres, presto, rabbitmq, redis, -s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, -snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, -weaviate, webhdfs, winrm, yandex, zendesk +pandas, papermill, password, pgvector, pinecone, pinot, postgres, presto, rabbitmq, redis, s3, s3fs, +salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, +spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, weaviate, +webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE Provider packages diff --git a/INSTALL b/INSTALL index a83a600307700..4b6da57ae840f 100644 --- a/INSTALL +++ b/INSTALL @@ -96,18 +96,18 @@ The list of available extras: # START EXTRAS HERE aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, -apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, -apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud, +apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.webhdfs, apprise, +arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, +cncf.kubernetes, cohere, common.io, common.sql, crypto, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, fab, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openai, openfaas, openlineage, opensearch, opsgenie, oracle, otel, pagerduty, -pandas, papermill, password, pgvector, pinecone, pinot, plexus, postgres, presto, rabbitmq, redis, -s3, s3fs, salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, -snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, -weaviate, webhdfs, winrm, yandex, zendesk +pandas, papermill, password, pgvector, pinecone, pinot, postgres, presto, rabbitmq, redis, s3, s3fs, +salesforce, samba, saml, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, +spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, weaviate, +webhdfs, winrm, yandex, zendesk # END EXTRAS HERE # For installing Airflow in development environments - see CONTRIBUTING.rst diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst index 5a9840519d997..311a65aadc085 100644 --- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst +++ b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst @@ -514,14 +514,15 @@ providers (and then remove all the code and documentation related to the provide The "removed: true" flag will cause the provider to be available for the following commands (note that such provider has to be explicitly added as selected to the package - such provider will not be included in -the available list of providers): +the available list of providers or when documentation is built unless --include-removed-providers +flag is used): * ``breeze build-docs`` * ``breeze release-management prepare-provider-documentation`` * ``breeze release-management prepare-provider-packages`` * ``breeze release-management publish-docs`` -For all those commands, release manager needs to specify such to-be-removed provider explicitly as extra -command during the release process. Except the changelog that needs to be maintained manually, all other -documentation (main page of the provider documentation, PyPI README), will be automatically updated -to include removal notice. +For all those commands, release manager needs to specify ``--include-removed-providers`` when all providers +are built or must add the provider id explicitly during the release process. +Except the changelog that needs to be maintained manually, all other documentation (main page of the provider +documentation, PyPI README), will be automatically updated to include removal notice. diff --git a/airflow/providers/apache/sqoop/provider.yaml b/airflow/providers/apache/sqoop/provider.yaml index e263cced9df91..d6bc9b6c60700 100644 --- a/airflow/providers/apache/sqoop/provider.yaml +++ b/airflow/providers/apache/sqoop/provider.yaml @@ -21,7 +21,8 @@ name: Apache Sqoop description: | `Apache Sqoop `__ -suspended: false +removed: true +suspended: true source-date-epoch: 1701983361 versions: - 4.2.0 diff --git a/airflow/providers/plexus/provider.yaml b/airflow/providers/plexus/provider.yaml index a7cb354ff4922..64752b5fd53cf 100644 --- a/airflow/providers/plexus/provider.yaml +++ b/airflow/providers/plexus/provider.yaml @@ -21,7 +21,8 @@ name: Plexus description: | `Plexus `__ -suspended: false +removed: true +suspended: true source-date-epoch: 1701983408 versions: - 3.4.0 diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index bb89dbf6b00fd..f50b8e2c349be 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -109,17 +109,17 @@ def test_get_provider_requirements(): def test_get_removed_providers(): # Modify it every time we schedule provider for removal or remove it - assert ["daskexecutor"] == get_removed_provider_ids() + assert ["apache.sqoop", "daskexecutor", "plexus"] == get_removed_provider_ids() def test_get_suspended_provider_ids(): # Modify it every time we suspend/resume provider - assert ["daskexecutor"] == get_suspended_provider_ids() + assert ["apache.sqoop", "daskexecutor", "plexus"] == get_suspended_provider_ids() def test_get_suspended_provider_folders(): # Modify it every time we suspend/resume provider - assert ["daskexecutor"] == get_suspended_provider_folders() + assert ["apache/sqoop", "daskexecutor", "plexus"] == get_suspended_provider_folders() @pytest.mark.parametrize( diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index fcda9e6ddc59f..082595312edf3 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -144,8 +144,6 @@ custom bash/python providers). +---------------------+-----------------------------------------------------+------------------------------------------------+ | apache.spark | ``pip install 'apache-airflow[apache.spark]'`` | All Spark related operators & hooks | +---------------------+-----------------------------------------------------+------------------------------------------------+ -| apache.sqoop | ``pip install 'apache-airflow[apache.sqoop]'`` | All Sqoop related operators & hooks | -+---------------------+-----------------------------------------------------+------------------------------------------------+ | apache.webhdfs | ``pip install 'apache-airflow[apache.webhdfs]'`` | HDFS hooks and operators | +---------------------+-----------------------------------------------------+------------------------------------------------+ @@ -204,8 +202,6 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | pinecone | ``pip install 'apache-airflow[pinecone]'`` | Pinecone Operators and Hooks | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ -| plexus | ``pip install 'apache-airflow[plexus]'`` | Plexus service of CoreScientific.com AI platform | -+---------------------+-----------------------------------------------------+-----------------------------------------------------+ | salesforce | ``pip install 'apache-airflow[salesforce]'`` | Salesforce hook | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | sendgrid | ``pip install 'apache-airflow[sendgrid]'`` | Send email using sendgrid | diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index c0eb6a6340b04..780009c5365e0 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -198,13 +198,6 @@ ], "excluded-python-versions": [] }, - "apache.sqoop": { - "deps": [ - "apache-airflow>=2.6.0" - ], - "cross-providers-deps": [], - "excluded-python-versions": [] - }, "apprise": { "deps": [ "apache-airflow>=2.6.0", @@ -762,14 +755,6 @@ "cross-providers-deps": [], "excluded-python-versions": [] }, - "plexus": { - "deps": [ - "apache-airflow>=2.6.0", - "arrow>=0.16.0" - ], - "cross-providers-deps": [], - "excluded-python-versions": [] - }, "postgres": { "deps": [ "apache-airflow-providers-common-sql>=1.3.1", diff --git a/images/breeze/output_release-management_generate-issue-content-providers.svg b/images/breeze/output_release-management_generate-issue-content-providers.svg index 9ee29fb1c755a..85d5b5206754c 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.svg +++ b/images/breeze/output_release-management_generate-issue-content-providers.svg @@ -143,13 +143,13 @@ breeze release-management generate-issue-content-providers                                                             [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |    -apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |     -exasol | fab | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |         -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |       -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...                        +apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io +| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | fab |         +facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure |      +microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  +opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce +| samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular |      +telegram | trino | vertica | weaviate | yandex | zendesk]...                                                           Generates content for issue to test the release. diff --git a/images/breeze/output_release-management_generate-issue-content-providers.txt b/images/breeze/output_release-management_generate-issue-content-providers.txt index 2a054a47eee1b..c6c792a2a4537 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.txt +++ b/images/breeze/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -016d26c8813cddffc28fa908328d48cc +ad4c012ebf783856bbe975670d2d641b diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg index f9625356af9d3..146d08ba7236c 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.svg +++ b/images/breeze/output_release-management_prepare-provider-documentation.svg @@ -173,13 +173,13 @@ breeze release-management prepare-provider-documentation                                                               [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |    -apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |     -exasol | fab | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |         -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |       -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...                        +apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io +| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | fab |         +facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure |      +microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  +opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce +| samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular |      +telegram | trino | vertica | weaviate | yandex | zendesk]...                                                           Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/images/breeze/output_release-management_prepare-provider-documentation.txt b/images/breeze/output_release-management_prepare-provider-documentation.txt index 8b1406bebbcab..8d78fd15a1990 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.txt +++ b/images/breeze/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -780840f3794a18723468b6e48d797649 +0e614af4878bb630f74a0109e7f9e037 diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg index 9ab974b45ca80..c97c4ea9363fe 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.svg +++ b/images/breeze/output_release-management_prepare-provider-packages.svg @@ -158,13 +158,13 @@ breeze release-management prepare-provider-packages                                                                    [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot |    -apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.io | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch |     -exasol | fab | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins |         -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | plexus |       -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | weaviate | yandex | zendesk]...                        +apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io +| common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | fab |         +facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure |      +microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  +opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | redis | salesforce +| samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular |      +telegram | trino | vertica | weaviate | yandex | zendesk]...                                                           Prepare sdist/whl packages of Airflow Providers. diff --git a/images/breeze/output_release-management_prepare-provider-packages.txt b/images/breeze/output_release-management_prepare-provider-packages.txt index 9dab1458739bc..56e6f06a82191 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.txt +++ b/images/breeze/output_release-management_prepare-provider-packages.txt @@ -1 +1 @@ -d2121184dc6c9c85c91b2bce84b4b3e2 +1e0fb496eb112c5f2eaaddca369d4bee diff --git a/images/breeze/output_sbom_generate-providers-requirements.svg b/images/breeze/output_sbom_generate-providers-requirements.svg index 926c8b925f7a0..dbc947ad26b4a 100644 --- a/images/breeze/output_sbom_generate-providers-requirements.svg +++ b/images/breeze/output_sbom_generate-providers-requirements.svg @@ -185,15 +185,15 @@ │--provider-idProvider id to generate the requirements for                                                  │ │(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid |  │ │apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin |      │ -│apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb |  │ -│asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql│ -│| databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol |   │ -│fab | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc |   │ +│apache.livy | apache.pig | apache.pinot | apache.spark | apprise | arangodb | asana |         │ +│atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql |      │ +│databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | fab │ +│| facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc |       │ │jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql│ │| neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty │ -│| papermill | pgvector | pinecone | plexus | postgres | presto | redis | salesforce | samba | │ -│segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | │ -│tabular | telegram | trino | vertica | weaviate | yandex | zendesk)                           │ +│| papermill | pgvector | pinecone | postgres | presto | redis | salesforce | samba | segment |│ +│sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | │ +│telegram | trino | vertica | weaviate | yandex | zendesk)                                     │ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported   │ │value to account for the most recent version of the provider                                  │ │(TEXT)                                                                                        │ diff --git a/images/breeze/output_sbom_generate-providers-requirements.txt b/images/breeze/output_sbom_generate-providers-requirements.txt index f997d309bd170..56eadeec24b79 100644 --- a/images/breeze/output_sbom_generate-providers-requirements.txt +++ b/images/breeze/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -e7ce233650bb20532d6d8d0740869fc9 +84d46887b3f47bc209014ec5cb26406c