From 3c859448b4f0d4e8dd1cd62d7e55ab1673446da5 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 29 Dec 2024 15:31:54 +0100 Subject: [PATCH] Add several pypi tests These tests are designed to make sure multiple selectors for the same package are preserved correctly at every stage of the process of merging dependencies from different sources. When parsing poetry dependencies, we introduce selectors early in the process, whereas for other approaches, we retain "extras" (separated by semicolons) for much of the merging process. Packages employing each strategy are used in the added tests. --- tests/test_pypi.py | 168 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/tests/test_pypi.py b/tests/test_pypi.py index b802e34db..e4aa923a3 100644 --- a/tests/test_pypi.py +++ b/tests/test_pypi.py @@ -682,6 +682,124 @@ def test_merge_pypi_sdist_metadata(): ) +def test_pypi_metadata_constraints_for_python_versions(): + config = Configuration(name="apache-airflow-providers-trino", + version="6.0.0") + pypi_metadata = get_pypi_metadata(config) + assert sorted(pypi_metadata["requires_dist"]) == sorted( + [ + 'apache-airflow-providers-common-sql>=1.20.0', + 'apache-airflow>=2.9.0', + 'pandas<2.2,>=1.5.3; python_version < "3.9"', + 'pandas<2.2,>=2.1.2; python_version >= "3.9"', + 'trino>=0.318.0', + 'apache-airflow-providers-google; extra == "google"', + 'apache-airflow-providers-openlineage; extra == "openlineage"' + ] + ) + + config = Configuration(name="databricks-sql-connector", version="3.7.0") + pypi_metadata = get_pypi_metadata(config) + assert sorted(pypi_metadata["requires_dist"]) == sorted( + [ + 'alembic<2.0.0,>=1.0.11; extra == "alembic"', + 'lz4<5.0.0,>=4.0.2', + 'numpy>=1.16.6; python_version >= "3.8" and python_version < "3.11"', + 'numpy>=1.23.4; python_version >= "3.11"', + 'oauthlib<4.0.0,>=3.1.0', + 'openpyxl<4.0.0,>=3.0.10', + 'pandas<2.3.0,>=1.2.5; python_version >= "3.8"', + 'pyarrow>=14.0.1', + 'requests<3.0.0,>=2.18.1', + 'sqlalchemy>=2.0.21; extra == "sqlalchemy" or extra == "alembic"', + 'thrift<0.21.0,>=0.16.0', + 'urllib3>=1.26', + ] + ) + + +def test_sdist_metadata_from_toml_project_dependencies(): + config = Configuration(name="apache-airflow-providers-trino", + version="6.0.0") + pypi_metadata = get_pypi_metadata(config) + sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config) + assert sorted(sdist_metadata["install_requires"]) == sorted( + [ + 'apache-airflow-providers-common-sql>=1.20.0', + 'apache-airflow>=2.9.0', + 'pandas>=1.5.3,<2.2;python_version<"3.9"', + 'pandas>=2.1.2,<2.2;python_version>="3.9"', + 'trino>=0.318.0', + 'python ~=3.9' + ] + ) + + +def test_sdist_metadata_from_toml_poetry_dependencies(): + config = Configuration(name="databricks-sql-connector", version="3.7.0") + pypi_metadata = get_pypi_metadata(config) + sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config) + assert sorted(sdist_metadata["install_requires"]) == sorted( + [ + 'python >=3.8.0,<4.0.0', + 'thrift >=0.16.0,<0.21.0', + 'pandas >=1.2.5,<2.3.0 # [py>=38]', + 'pyarrow >=14.0.1', + 'lz4 >=4.0.2,<5.0.0', + 'requests >=2.18.1,<3.0.0', + 'oauthlib >=3.1.0,<4.0.0', + 'numpy >=1.16.6 # [py>=38 and py<311]', + 'numpy >=1.23.4 # [py>=311]', + 'openpyxl >=3.0.10,<4.0.0', + 'urllib3 >=1.26' + ] + ) + + +def test_merge_pypi_sdist_metadata_from_toml(): + # tests merging pyproject.toml dependencies from poetry with pypi data, + # including multiple numpy constraints with python version selectors + config = Configuration(name="databricks-sql-connector", version="3.7.0") + pypi_metadata = get_pypi_metadata(config) + sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config) + merged_data = merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata, + config) + assert sorted(merged_data["requires_dist"]) == sorted( + [ + 'python >=3.8.0,<4.0.0', + 'thrift >=0.16.0,<0.21.0', + 'pandas >=1.2.5,<2.3.0 # [py>=38]', + 'pyarrow >=14.0.1', + 'lz4 >=4.0.2,<5.0.0', + 'requests >=2.18.1,<3.0.0', + 'oauthlib >=3.1.0,<4.0.0', + 'numpy >=1.16.6 # [py>=38 and py<311]', + 'numpy >=1.23.4 # [py>=311]', + 'openpyxl >=3.0.10,<4.0.0', + 'urllib3 >=1.26' + ] + ) + + # tests merging pyproject.toml project dependencies with pypi data, + # including multiple pandas constraints with python version selectors + config = Configuration(name="apache-airflow-providers-trino", + version="6.0.0") + pypi_metadata = get_pypi_metadata(config) + sdist_metadata = get_sdist_metadata(pypi_metadata["sdist_url"], config) + merged_data = merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata, + config) + assert sorted(merged_data["requires_dist"]) == sorted( + [ + 'apache-airflow-providers-common-sql>=1.20.0', + 'apache-airflow>=2.9.0', + 'pandas>=1.5.3,<2.2;python_version<"3.9"', + 'pandas>=2.1.2,<2.2;python_version>="3.9"', + 'trino>=0.318.0', + 'python ~=3.9' + ] + ) + + def test_update_requirements_with_pin(): req = { "build": ["<{ compiler('c') }}"], @@ -787,6 +905,56 @@ def test_run_requirements_sdist(): ] ) + # a more complex example with selectors + config = Configuration(name="apache-airflow-providers-trino", + version="6.0.0") + recipe = GrayskullFactory.create_recipe("pypi", config) + assert "noarch" not in recipe["build"] + selectors = { + "python >=3.9,<4.dev0": None, + "apache-airflow-providers-common-sql >=1.20.0": None, + "apache-airflow >=2.9.0": None, + "pandas >=1.5.3,<2.2": "[py<39]", + "pandas >=2.1.2,<2.2": "[py>=39]", + "trino-python-client >=0.318.0": None + } + + for dep in recipe["requirements"]["run"]: + dep_str = str(dep) + assert dep_str in selectors + selector = selectors[dep_str] + if dep.inline_comment is None: + assert selector is None + else: + assert str(dep.inline_comment) == selector + + # another example with selectors + config = Configuration(name="databricks-sql-connector", version="3.7.0") + recipe = GrayskullFactory.create_recipe("pypi", config) + assert "noarch" not in recipe["build"] + selectors = { + "python >=3.8.0,<4.0.0": None, + "thrift >=0.16.0,<0.21.0": None, + "pandas >=1.2.5,<2.3.0": "[py>=38]", + "pyarrow >=14.0.1": None, + "lz4 >=4.0.2,<5.0.0": None, + "requests >=2.18.1,<3.0.0": None, + "oauthlib >=3.1.0,<4.0.0": None, + "numpy >=1.16.6": "[py>=38 and py<311]", + "numpy >=1.23.4": "[py>=311]", + "openpyxl >=3.0.10,<4.0.0": None, + "urllib3 >=1.26": None + } + + for dep in recipe["requirements"]["run"]: + dep_str = str(dep) + assert dep_str in selectors + selector = selectors[dep_str] + if dep.inline_comment is None: + assert selector is None + else: + assert str(dep.inline_comment) == selector + def test_format_host_requirements(): assert sorted(format_dependencies(["setuptools>=40.0", "pkg2"], "pkg1")) == sorted(