diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7ca8a60..09ab7cd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,14 +8,14 @@ on: jobs: lint: - runs-on: ubuntu-20.04 # Latest version supporting Python 3.6 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: 3.9 - name: Install dependencies run: | pip install --upgrade pip @@ -29,11 +29,11 @@ jobs: mypy --config-file setup.cfg build: - runs-on: ubuntu-20.04 # Latest version supporting Python 3.6 + runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 @@ -62,7 +62,7 @@ jobs: - name: Build the docker-compose stack run: | export PYTHON_VERSION=3.9 - docker-compose -f ./examples/spark-with-S3/docker-compose.yml up -d + docker compose -f ./examples/spark-with-S3/docker-compose.yml up -d - name: Check running containers run: docker ps -a @@ -70,35 +70,16 @@ jobs: - name: Run spark Job run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.9 3.2.2 - standalone_spark2_with_S3: - runs-on: ubuntu-20.04 # Latest version supporting Python 3.6 - - steps: - - uses: actions/checkout@v2 - - - name: Build spark-docker - run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=2.4.2 --build-arg PYTHON_VERSION=3.6.9 - - - name: Build the docker-compose stack - run: | - export PYTHON_VERSION=3.6 - docker-compose -f ./examples/spark-with-S3/docker-compose.yml up -d - - - name: Check running containers - run: docker ps -a - - - name: Run spark Job - run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.6 2.4.2 # hadoop_hdfs: - # runs-on: ubuntu-20.04 # Latest version supporting Python 3.6 + # runs-on: ubuntu-latest # steps: # - uses: actions/checkout@v2 - # - name: Set up Python 3.6 + # - name: Set up Python 3.9 # uses: actions/setup-python@v2 # with: - # python-version: 3.6 + # python-version: 3.9 # - name: Install hadoop-test-cluster # run: | @@ -111,8 +92,8 @@ jobs: # - name: Start Job # run: | # # for the hack with script .. see https://github.com/actions/runner/issues/241#issuecomment-577360161 - # # the prebuild image only contains a conda install, we also install python 3.6.10 - # # to avoid sharing files on the worker node we copy the python3.6 install script via hdfs to worker /tmp folder + # # the prebuild image only contains a conda install, we also install python + # # to avoid sharing files on the worker node we copy the python install script via hdfs to worker /tmp folder # script -e -c "htcluster exec -u root -s edge -- chown -R testuser /home/testuser && \ # htcluster exec -u root -s edge -- /home/testuser/cluster-pack/tests/integration/install_python.sh && \ # htcluster exec -u root -s edge -- hdfs dfs -put /home/testuser/cluster-pack/tests/integration/install_python.sh hdfs:///tmp && \ @@ -122,11 +103,11 @@ jobs: # htcluster exec -s edge -- /home/testuser/cluster-pack/tests/integration/hadoop_hdfs_tests.sh" conda: - runs-on: ubuntu-20.04 # Latest version supporting Python 3.6 + runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.9] + python-version: [3.9] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 2704e87..f797b3e 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ $ pip install . ## Prerequisites -cluster-pack supports Python ≥3.6. +cluster-pack supports Python ≥3.7. ## Features diff --git a/cluster_pack/conda.py b/cluster_pack/conda.py index 35dbf3a..0081758 100644 --- a/cluster_pack/conda.py +++ b/cluster_pack/conda.py @@ -55,7 +55,7 @@ def get_or_create_conda_env(project_env_name: str = None, spec_file: str = None) spec_file]) else: process.call( - [conda_path, "create", "-n", project_env_name, "python=3.6"]) + [conda_path, "create", "-n", project_env_name, "python=3.9"]) project_env_path = [env for env in _list_envs(conda_path) if os.path.basename(env) == project_env_name][0] diff --git a/examples/skein-project/setup.py b/examples/skein-project/setup.py index ebea66d..28851d3 100644 --- a/examples/skein-project/setup.py +++ b/examples/skein-project/setup.py @@ -4,7 +4,7 @@ name='skein_project', version='0.0.1', packages=setuptools.find_packages(), - python_requires=">=3.6", + python_requires=">=3.7", install_requires=[ "skein", "numpy", diff --git a/examples/skein-project/skein_example.sh b/examples/skein-project/skein_example.sh index 22b876f..91f1cc5 100644 --- a/examples/skein-project/skein_example.sh +++ b/examples/skein-project/skein_example.sh @@ -1,7 +1,7 @@ #!/bin/bash # install venv -python3.6 -m venv skein_project_env +python3.9 -m venv skein_project_env . skein_project_env/bin/activate pip install --upgrade pip setuptools pip install -e . diff --git a/examples/spark-with-S3/README.md b/examples/spark-with-S3/README.md index 1cf8f5b..3fd3788 100644 --- a/examples/spark-with-S3/README.md +++ b/examples/spark-with-S3/README.md @@ -13,7 +13,7 @@ docker build -t spark-docker ./examples/spark-with-S3 2) Start standalone Spark & local S3 containers (using [minio](https://min.io/)) ```bash -docker-compose -f ./examples/spark-with-S3/docker-compose.yml up -d +docker-compose -f ./examples/spark-with-S3/docker compose.yml up -d ``` ### Quick run diff --git a/examples/spark-with-S3/scripts/run_spark_example.sh b/examples/spark-with-S3/scripts/run_spark_example.sh index b25e6bf..4849b28 100755 --- a/examples/spark-with-S3/scripts/run_spark_example.sh +++ b/examples/spark-with-S3/scripts/run_spark_example.sh @@ -5,10 +5,6 @@ $1 -m venv /tmp/pyspark_env pip install -U pip setuptools wheel pip install pypandoc<1.8 pip install s3fs pandas pyspark==$2 -if [ $1 == "python3.6" ] -then - pip install 'pandas<1.0.0' pyarrow==0.14.1 -fi pip install -e /cluster-pack curr_dir=$(dirname "$0") diff --git a/requirements.txt b/requirements.txt index 42446d8..6a71d04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ cloudpickle pex==2.1.137 conda-pack -pip>=18.1 +pip>=22.0 pyarrow<16.0.0 fire types-setuptools diff --git a/scripts/hadoop_hdfs_test.ps1 b/scripts/hadoop_hdfs_test.ps1 index abaa938..b01b44a 100644 --- a/scripts/hadoop_hdfs_test.ps1 +++ b/scripts/hadoop_hdfs_test.ps1 @@ -3,7 +3,7 @@ # run from root dir cluster-pack scripts/hadoop_hdfs_tests.ps1 pip install hadoop-test-cluster htcluster startup --image cdh5 --mount .:cluster-pack -# to avoid sharing files on the worker node we copy the python3.6 install script via hdfs to worker /tmp folder +# to avoid sharing files on the worker node we copy the python install script via hdfs to worker /tmp folder htcluster exec -u root -s edge -- chown -R testuser /home/testuser htcluster exec -u root -s edge -- /home/testuser/cluster-pack/tests/integration/install_python.sh htcluster exec -u root -s edge -- hdfs dfs -put /home/testuser/cluster-pack/tests/integration/install_python.sh hdfs:///tmp diff --git a/setup.py b/setup.py index b12c731..f2f1af1 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,11 @@ def _read_reqs(relpath): "Environment :: Console", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries" ] @@ -44,7 +45,7 @@ def _read_reqs(relpath): version=versioneer.get_version(), install_requires=REQUIREMENTS, tests_require=["pytest"], - python_requires=">=3.6", + python_requires=">=3.7", maintainer="Criteo", maintainer_email="github@criteo.com", description=DESCRIPTION, diff --git a/tests-requirements.txt b/tests-requirements.txt index f57070d..3390413 100644 --- a/tests-requirements.txt +++ b/tests-requirements.txt @@ -5,4 +5,4 @@ pytest pyflakes==2.4.0 pylama mypy -protobuf!=4.21.0 +numpy<2 # breaking changes from numpy 2+, can be removed when all dependencies are compatible \ No newline at end of file diff --git a/tests/integration/hadoop_hdfs_tests.sh b/tests/integration/hadoop_hdfs_tests.sh index acf034e..a51258b 100755 --- a/tests/integration/hadoop_hdfs_tests.sh +++ b/tests/integration/hadoop_hdfs_tests.sh @@ -4,7 +4,7 @@ set -e pushd cluster-pack rm -rf ~/venv - python3.6 -m venv ~/venv + python3.9 -m venv ~/venv source ~/venv/bin/activate pip install -U wheel pip setuptools pip install -e . diff --git a/tests/integration/install_python.sh b/tests/integration/install_python.sh index 8a1b6cb..8b346fa 100755 --- a/tests/integration/install_python.sh +++ b/tests/integration/install_python.sh @@ -2,18 +2,18 @@ set -e -if [ -d "/usr/src/Python-3.6.10" ] +if [ -d "/usr/src/Python-3.9.15" ] then - echo "Python already installed in /usr/src/Python-3.6.10" + echo "Python already installed in /usr/src/Python-3.9.15" exit fi # install python interpreter globally yum install -y wget gcc openssl-devel bzip2-devel libffi-devel pushd /usr/src -wget https://www.python.org/ftp/python/3.6.10/Python-3.6.10.tgz -tar xzf Python-3.6.10.tgz -pushd Python-3.6.10 +wget https://www.python.org/ftp/python/3.9.15/Python-3.9.15.tgz +tar xzf Python-3.9.15.tgz +pushd Python-3.9.15 ./configure --enable-optimizations make altinstall -rm /usr/src/Python-3.6.10.tgz +rm /usr/src/Python-3.9.15.tgz diff --git a/tests/resources/conda.yaml b/tests/resources/conda.yaml index bd06343..ad2d025 100644 --- a/tests/resources/conda.yaml +++ b/tests/resources/conda.yaml @@ -4,5 +4,5 @@ name: tutorial channels: - defaults dependencies: - - python=3.6 + - python=3.9 - botocore==1.17.12 diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 9420ccf..addd833 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -18,6 +18,7 @@ MYARCHIVE_FILENAME = "myarchive.pex" MYARCHIVE_METADATA = "myarchive.json" VARNAME = 'VARNAME' +PINNED_VERSIONS_FOR_COMPATIBILITY_ISSUE = {"numpy": "numpy<2"} def test_get_virtualenv_name(): @@ -37,7 +38,7 @@ def test_get_empty_editable_requirements(): with tempfile.TemporaryDirectory() as tempdir: _create_venv(tempdir) subprocess.check_call([f"{tempdir}/bin/python", "-m", "pip", "install", - "cloudpickle", _get_editable_package_name(), "pip==18.1"]) + "cloudpickle", _get_editable_package_name(), "pip==22.0"]) editable_requirements = packaging._get_editable_requirements(f"{tempdir}/bin/python") assert len(editable_requirements) == 0 @@ -46,7 +47,7 @@ def test_get_empty_non_editable_requirements(): with tempfile.TemporaryDirectory() as tempdir: _create_venv(tempdir) subprocess.check_call([f"{tempdir}/bin/python", "-m", "pip", "install", - "-e", _get_editable_package_name(), "pip==18.1"]) + "-e", _get_editable_package_name(), "pip==22.0"]) non_editable_requirements = packaging.get_non_editable_requirements( f"{tempdir}/bin/python") assert len(non_editable_requirements) == 2 @@ -75,7 +76,6 @@ def test__get_editable_requirements_for_src_layout(): assert "user_lib2" in pkg_names -@pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher") def test__get_editable_requirements_withpip23(): with tempfile.TemporaryDirectory() as tempdir: _create_venv(tempdir) @@ -101,7 +101,7 @@ def _create_venv(tempdir: str): subprocess.check_call([sys.executable, "-m", "venv", f"{tempdir}"]) -def _pip_install(tempdir: str, pip_version: str = "18.1", use_src_layout: bool = False): +def _pip_install(tempdir: str, pip_version: str = "22.0", use_src_layout: bool = False): subprocess.check_call([f"{tempdir}/bin/python", "-m", "pip", "install", "cloudpickle", f"pip=={pip_version}"]) pkg = (_get_editable_package_name_src_layout() if use_src_layout @@ -123,7 +123,7 @@ def test_get_current_pex_filepath(): with tempfile.TemporaryDirectory() as tempdir: path_to_pex = f"{tempdir}/out.pex" packaging.pack_in_pex( - ["numpy"], + [PINNED_VERSIONS_FOR_COMPATIBILITY_ISSUE['numpy']], path_to_pex, # make isolated pex from current pytest virtual env pex_inherit_path="false") @@ -215,7 +215,7 @@ def test_pack_in_pex(pyarrow_version, expectation): def test_pack_in_pex_with_allow_large(): with tempfile.TemporaryDirectory() as tempdir: - requirements = ["pyarrow==6.0.1"] + requirements = [PINNED_VERSIONS_FOR_COMPATIBILITY_ISSUE['numpy'], "pyarrow==6.0.1"] packaging.pack_in_pex( requirements, f"{tempdir}/out.pex", @@ -241,7 +241,7 @@ def test_pack_in_pex_with_allow_large(): def test_pack_in_pex_with_include_tools(): with tempfile.TemporaryDirectory() as tempdir: - requirements = ["pyarrow==6.0.1"] + requirements = [PINNED_VERSIONS_FOR_COMPATIBILITY_ISSUE['numpy'], "pyarrow==6.0.1"] packaging.pack_in_pex( requirements, f"{tempdir}/out.pex", @@ -310,12 +310,8 @@ def test_pack_in_pex_with_large_correctly_retrieves_zip_archive(is_large_pex, pa def test_pack_in_pex_with_additional_repo(): - if sys.version_info.minor == 6: - # dependency issue with available pytorch on https://download.pytorch.org/whl/cpu - return - with tempfile.TemporaryDirectory() as tempdir: - requirements = ["setuptools", "torch", + requirements = ["torch", "typing-extensions<=3.7.4.3; python_version<'3.8'", "networkx<2.6; python_version<'3.9'"] packaging.pack_in_pex( diff --git a/tests/test_uploader.py b/tests/test_uploader.py index ae1e299..72ec841 100644 --- a/tests/test_uploader.py +++ b/tests/test_uploader.py @@ -1,3 +1,5 @@ +import re +import subprocess import sys import contextlib import json @@ -77,11 +79,6 @@ def test_update_no_metadata(): 'platform': 'fake_platform2', 'python_version': '3.9.10'}, False), - pytest.param(["a==2.0", "b==1.0"], - {'package_installed': ["a==2.0", "b==1.0"], - 'platform': 'fake_platform', - 'python_version': '3.6.8'}, - False), ]) @mock.patch(f'{MODULE_TO_TEST}.platform.platform') @mock.patch(f'{MODULE_TO_TEST}.sys') @@ -446,6 +443,21 @@ def test__unique_filename(spec_file, expected): assert expected == uploader._unique_filename(spec_file, packaging.PEX_PACKER) +def get_latest_pip_version() -> str: + p = subprocess.Popen("pip index versions pip", shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + for line in p.stdout.readlines(): + result = re.match(r"pip \((.+)\)", line.decode("utf-8")) + if result: + return result.group(1) + return None + + +def test_latest_pip(): + + assert get_latest_pip_version() is not None + + def test_format_pex_requirements(): with tempfile.TemporaryDirectory() as tempdir: requirements = ["pipdeptree==2.0.0", "six==1.15.0"] @@ -456,8 +468,8 @@ def test_format_pex_requirements(): pex_inherit_path="false") pex_info = PexInfo.from_pex(f"{tempdir}/out.pex") cleaned_requirements = uploader._format_pex_requirements(pex_info) - pip_version = 'pip==21.3.1' if sys.version_info.minor == 6 else 'pip==24.0' - assert [pip_version, 'pipdeptree==2.0.0', 'six==1.15.0'] == cleaned_requirements + assert [f'pip=={get_latest_pip_version()}', + 'pipdeptree==2.0.0', 'six==1.15.0'] == cleaned_requirements @pytest.mark.parametrize("req, expected", [