Add option --version-suffix-for-local (apache#43769)

contributing-docs/11_provider_packages.rst

@@ -118,29 +118,60 @@ in development mode - then capabilities of your provider will be discovered by a
 the provider among other providers in ``airflow providers`` command output.
 
 
-Local Development Release of a Specific Provider
+Local Release of a Specific Provider
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 When you develop a provider, you can release it locally and test it in your Airflow environment. This should
-be accomplished using breeze. Choose a suffix for the release such as "dev1" and run the breeze build for
+be accomplished using breeze. Choose a suffix for the release such as "patch.asb.1" and run the breeze build for
 that provider. Remember Provider IDs use a dot ('.') for directory separators so the Provider ID for the
-Microsoft Azure provider is 'microsoft.azure'. This can be provided in the PACKAGE_LIST environment variable
-or passed on the command line.
+Microsoft Azure provider is 'microsoft.azure'. The provider IDs to build can be provided in the PACKAGE_LIST
+environment variable or passed on the command line.
 
-``export PACKAGE_LIST=microsoft.azure``
+.. code-block:: bash
+
+     export PACKAGE_LIST=microsoft.azure
 
 Then build the provider (you don't need to pass the package ID if you set the environment variable above):
 
-```bash
-breeze release-management prepare-provider-packages \
-    --package-format both --version-suffix-for-pypi=dev1 \
-    --skip-tag-check microsoft.azure
-```
+.. code-block:: bash
+
+    breeze release-management prepare-provider-packages \
+        --package-format both \
+        --version-suffix-for-local=patch.asb.1 \
+        microsoft.azure
+
 
 Finally, copy the wheel file from the dist directory to the a directory your airflow deployment can use.
 If this is ~/airflow/test-airflow/local_providers, you can use the following command:
 
-``cp dist/apache_airflow_providers_microsoft_azure-10.5.2+dev1-py3-none-any.whl ~/airflow/test-airflow/local_providers/``
+``cp dist/apache_airflow_providers_microsoft_azure-10.5.2+patch.asb.1-none-any.whl ~/airflow/test-airflow/local_providers/``
+
+If you want to build a local version of a version already released to PyPI, such as rc1, then you can combine
+the PyPI suffix flag --version-suffix-for-pypi with the local suffix flag --version-suffix-for-local. For example:
+
+.. code-block:: bash
+
+    breeze release-management prepare-provider-packages \
+        --package-format both \
+        --version-suffix-for-pypi rc1 \
+        --version-suffix-for-local=patch.asb.1 \
+        microsoft.azure
+
+
+The above would result in a wheel file
+
+    apache_airflow_providers_microsoft_azure-10.5.2rc1+patch.asb.1-py3-none-any.whl
+
+Builds using a local suffix will not check to see if a release has already been made. This is useful for testing.
+
+Local versions can also be built using the version-suffix-for-pypi flag although using the version-suffix-for-local
+flag is preferred. To build with the version-suffix-for-pypi flag, use the following command:
+
+.. code-block:: bash
+
+    breeze release-management prepare-provider-packages \
+        --package-format both --version-suffix-for-pypi=dev1 \
+        --skip-tag-check microsoft.azure
 
 
 Naming Conventions for provider packages

dev/breeze/doc/images/output_release-management_prepare-provider-packages.txt

@@ -1 +1 @@
-a781b53f55fe962ebab27068bcd96e44
+85b9590e16c1986679675b5e605844da

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -157,7 +157,12 @@
     run_command,
 )
 from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
-from airflow_breeze.utils.version_utils import get_latest_airflow_version, get_latest_helm_chart_version
+from airflow_breeze.utils.version_utils import (
+    create_package_version,
+    get_latest_airflow_version,
+    get_latest_helm_chart_version,
+    is_local_package_version,
+)
 from airflow_breeze.utils.versions import is_pre_release
 from airflow_breeze.utils.virtualenv_utils import create_pip_command, create_venv
 
@@ -864,6 +869,15 @@ def basic_provider_checks(provider_package_id: str) -> dict[str, Any]:
     is_flag=True,
     help="Skip checking if the tag already exists in the remote repository",
 )
+@click.option(
+    "--version-suffix-for-local",
+    default=None,
+    show_default=False,
+    help="Version suffix for local builds. Do not provide the leading plus sign ('+'). The suffix must "
+    "contain only ascii letters, numbers, and periods. The first character must be an ascii letter or number "
+    "and the last character must be an ascii letter or number. Note: the local suffix will be appended after "
+    "the PyPi suffix if both are provided.",
+)
 @click.option(
     "--skip-deleting-generated-files",
     default=False,
@@ -904,6 +918,7 @@ def prepare_provider_packages(
     skip_deleting_generated_files: bool,
     skip_tag_check: bool,
     version_suffix_for_pypi: str,
+    version_suffix_for_local: str,
 ):
     perform_environment_checks()
     fix_ownership_using_docker()
@@ -926,7 +941,8 @@ def prepare_provider_packages(
         include_removed=include_removed_providers,
         include_not_ready=include_not_ready_providers,
     )
-    if not skip_tag_check:
+    package_version = create_package_version(version_suffix_for_pypi, version_suffix_for_local)
+    if not skip_tag_check and not is_local_package_version(package_version):
         run_command(["git", "remote", "rm", "apache-https-for-providers"], check=False, stderr=DEVNULL)
         make_sure_remote_apache_exists_and_fetch(github_repository=github_repository)
     success_packages = []
@@ -939,7 +955,6 @@ def prepare_provider_packages(
         shutil.rmtree(DIST_DIR, ignore_errors=True)
         DIST_DIR.mkdir(parents=True, exist_ok=True)
     for provider_id in packages_list:
-        package_version = version_suffix_for_pypi
         try:
             basic_provider_checks(provider_id)
             if not skip_tag_check:

dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py

@@ -214,6 +214,7 @@
                 "--skip-deleting-generated-files",
                 "--skip-tag-check",
                 "--version-suffix-for-pypi",
+                "--version-suffix-for-local",
                 "--package-list",
             ],
         }

dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py

@@ -39,6 +39,7 @@
 )
 from airflow_breeze.utils.path_utils import AIRFLOW_PROVIDERS_SRC, AIRFLOW_SOURCES_ROOT
 from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.version_utils import is_local_package_version
 
 LICENCE_RST = """
 .. Licensed to the Apache Software Foundation (ASF) under one
@@ -161,8 +162,11 @@ def should_skip_the_package(provider_id: str, version_suffix: str) -> tuple[bool
     For RC and official releases we check if the "officially released" version exists
     and skip the released if it was. This allows to skip packages that have not been
     marked for release in this wave. For "dev" suffixes, we always build all packages.
+    A local version of an RC release will always be built.
     """
-    if version_suffix != "" and not version_suffix.startswith("rc"):
+    if version_suffix != "" and (
+        not version_suffix.startswith("rc") or is_local_package_version(version_suffix)
+    ):
         return False, version_suffix
     if version_suffix == "":
         current_tag = get_latest_provider_tag(provider_id, "")

dev/breeze/src/airflow_breeze/utils/packages.py

@@ -48,6 +48,7 @@
     get_provider_yaml_paths,
 )
 from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.version_utils import remove_local_version_suffix
 from airflow_breeze.utils.versions import get_version_tag, strip_leading_zeros_from_version
 
 MIN_AIRFLOW_VERSION = "2.8.0"
@@ -420,7 +421,9 @@ def get_dist_package_name_prefix(provider_id: str) -> str:
 
 
 def apply_version_suffix(install_clause: str, version_suffix: str) -> str:
-    if install_clause.startswith("apache-airflow") and ">=" in install_clause and version_suffix:
+    # Need to resolve a version suffix based on PyPi versions, but can ignore local version suffix.
+    pypi_version_suffix = remove_local_version_suffix(version_suffix)
+    if pypi_version_suffix and install_clause.startswith("apache-airflow") and ">=" in install_clause:
         # Applies version suffix to the apache-airflow and provider package dependencies to make
         # sure that pre-release versions have correct limits - this address the issue with how
         # pip handles pre-release versions when packages are pre-release and refer to each other - we
@@ -429,6 +432,8 @@ def apply_version_suffix(install_clause: str, version_suffix: str) -> str:
         # For example `apache-airflow-providers-fab==2.0.0.dev0` should refer to
         # `apache-airflow>=2.9.0.dev0` and not `apache-airflow>=2.9.0` because both packages are
         # released together and >= 2.9.0 is not correct reference for 2.9.0.dev0 version of Airflow.
+        # This assumes a local release, one where the suffix starts with a plus sign, uses the last
+        # version of the dependency, so it is not necessary to add the suffix to the dependency.
         prefix, version = install_clause.split(">=")
         # If version has a upper limit (e.g. ">=2.10.0,<3.0"), we need to cut this off not to fail
         if "," in version:
@@ -437,9 +442,9 @@ def apply_version_suffix(install_clause: str, version_suffix: str) -> str:
 
         base_version = Version(version).base_version
         # always use `pre-release`+ `0` as the version suffix
-        version_suffix = version_suffix.rstrip("0123456789") + "0"
+        pypi_version_suffix = pypi_version_suffix.rstrip("0123456789") + "0"
 
-        target_version = Version(str(base_version) + "." + version_suffix)
+        target_version = Version(str(base_version) + "." + pypi_version_suffix)
         return prefix + ">=" + str(target_version)
     return install_clause
 
@@ -590,6 +595,27 @@ def get_cross_provider_dependent_packages(provider_package_id: str) -> list[str]
     return PROVIDER_DEPENDENCIES[provider_package_id]["cross-providers-deps"]
 
 
+def format_version_suffix(version_suffix: str) -> str:
+    """
+    Formats the version suffix by adding a dot prefix unless it is a local prefix. If no version suffix is
+    passed in, an empty string is returned.
+
+    Args:
+        version_suffix (str): The version suffix to be formatted.
+
+    Returns:
+        str: The formatted version suffix.
+
+    """
+    if version_suffix:
+        if "." == version_suffix[0] or "+" == version_suffix[0]:
+            return version_suffix
+        else:
+            return f".{version_suffix}"
+    else:
+        return ""
+
+
 def get_provider_jinja_context(
     provider_id: str,
     current_release_version: str,
@@ -609,7 +635,7 @@ def get_provider_jinja_context(
         "FULL_PACKAGE_NAME": provider_details.full_package_name,
         "RELEASE": current_release_version,
         "RELEASE_NO_LEADING_ZEROS": release_version_no_leading_zeros,
-        "VERSION_SUFFIX": f".{version_suffix}" if version_suffix else "",
+        "VERSION_SUFFIX": format_version_suffix(version_suffix),
         "PIP_REQUIREMENTS": get_provider_requirements(provider_details.provider_id),
         "PROVIDER_DESCRIPTION": provider_details.provider_description,
         "INSTALL_REQUIREMENTS": get_install_requirements(

dev/breeze/src/airflow_breeze/utils/version_utils.py

@@ -35,3 +35,57 @@ def get_latest_airflow_version():
     response.raise_for_status()
     latest_released_version = response.json()["info"]["version"]
     return latest_released_version
+
+
+def create_package_version(version_suffix_for_pypi: str, version_suffix_for_local: str) -> str:
+    """
+    Creates a package version by combining the version suffix for PyPI and the version suffix for local. If
+    either one is an empty string, it is ignored. If the local suffix does not have a leading plus sign,
+    the leading plus sign will be added.
+
+    Args:
+        version_suffix_for_pypi (str): The version suffix for PyPI.
+        version_suffix_for_local (str): The version suffix for local.
+
+    Returns:
+        str: The combined package version.
+
+    """
+    # if there is no local version suffix, return the PyPi version suffix
+    if not version_suffix_for_local:
+        return version_suffix_for_pypi
+
+    # ensure the local version suffix starts with a plus sign
+    if version_suffix_for_local[0] != "+":
+        version_suffix_for_local = "+" + version_suffix_for_local
+
+    # if there is a PyPi version suffix, return the combined version. Otherwise just return the local version.
+    if version_suffix_for_pypi:
+        return version_suffix_for_pypi + version_suffix_for_local
+    else:
+        return version_suffix_for_local
+
+
+def remove_local_version_suffix(version_suffix: str) -> str:
+    if "+" in version_suffix:
+        return version_suffix.split("+")[0]
+    else:
+        return version_suffix
+
+
+def is_local_package_version(version_suffix: str) -> bool:
+    """
+    Check if the given version suffix is a local version suffix. A local version suffix will contain a
+    plus sign ('+'). This function does not guarantee that the version suffix is a valid local version suffix.
+
+    Args:
+        version_suffix (str): The version suffix to check.
+
+    Returns:
+        bool: True if the version suffix contains a '+', False otherwise. Please note this does not
+        guarantee that the version suffix is a valid local version suffix.
+    """
+    if version_suffix and ("+" in version_suffix):
+        return True
+    else:
+        return False