Update python versions and fix various github actions

.github/workflows/main.yml

@@ -8,14 +8,14 @@ on:
 
 jobs:
   lint:
-    runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.6
+    - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6
+        python-version: 3.9
     - name: Install dependencies
       run: |
         pip install --upgrade pip
@@ -29,11 +29,11 @@ jobs:
         mypy --config-file setup.cfg
 
   build:
-    runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
+    runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]   
+        python-version: [3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -62,43 +62,24 @@ jobs:
     - name: Build the docker-compose stack
       run: |
         export PYTHON_VERSION=3.9
-        docker-compose -f ./examples/spark-with-S3/docker-compose.yml up -d
+        docker compose -f ./examples/spark-with-S3/docker-compose.yml up -d
 
     - name: Check running containers
       run: docker ps -a
 
     - name: Run spark Job
       run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.9 3.2.2
 
-  standalone_spark2_with_S3:
-    runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build spark-docker
-      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=2.4.2 --build-arg PYTHON_VERSION=3.6.9
-
-    - name: Build the docker-compose stack
-      run: |
-        export PYTHON_VERSION=3.6
-        docker-compose -f ./examples/spark-with-S3/docker-compose.yml up -d
-
-    - name: Check running containers
-      run: docker ps -a
-
-    - name: Run spark Job
-      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.6 2.4.2
   # hadoop_hdfs:
-  #   runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
+  #   runs-on: ubuntu-latest
 
   #   steps:
   #   - uses: actions/checkout@v2
 
-  #   - name: Set up Python 3.6
+  #   - name: Set up Python 3.9
   #     uses: actions/setup-python@v2
   #     with:
-  #       python-version: 3.6
+  #       python-version: 3.9
 
   #   - name: Install hadoop-test-cluster
   #     run: |
@@ -111,8 +92,8 @@ jobs:
   #   - name: Start Job
   #     run: |
   #       # for the hack with script .. see https://github.com/actions/runner/issues/241#issuecomment-577360161
-  #       # the prebuild image only contains a conda install, we also install python 3.6.10
-  #       # to avoid sharing files on the worker node we copy the python3.6 install script via hdfs to worker /tmp folder
+  #       # the prebuild image only contains a conda install, we also install python
+  #       # to avoid sharing files on the worker node we copy the python install script via hdfs to worker /tmp folder
   #       script -e -c "htcluster exec -u root -s edge -- chown -R testuser /home/testuser && \
   #                     htcluster exec -u root -s edge -- /home/testuser/cluster-pack/tests/integration/install_python.sh && \                    
   #                     htcluster exec -u root -s edge -- hdfs dfs -put /home/testuser/cluster-pack/tests/integration/install_python.sh hdfs:///tmp && \
@@ -122,11 +103,11 @@ jobs:
   #                     htcluster exec -s edge -- /home/testuser/cluster-pack/tests/integration/hadoop_hdfs_tests.sh"
 
   conda:
-    runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
+    runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        python-version: [3.6, 3.9]   
+        python-version: [3.9]
 
     steps:
     - uses: actions/checkout@v2

README.md

@@ -28,7 +28,7 @@ $ pip install .
 
 ## Prerequisites
 
-cluster-pack supports Python ≥3.6.
+cluster-pack supports Python ≥3.7.
 
 ## Features
 

cluster_pack/conda.py

@@ -55,7 +55,7 @@ def get_or_create_conda_env(project_env_name: str = None, spec_file: str = None)
                           spec_file])
         else:
             process.call(
-                [conda_path, "create", "-n", project_env_name, "python=3.6"])
+                [conda_path, "create", "-n", project_env_name, "python=3.9"])
 
     project_env_path = [env for env in _list_envs(conda_path)
                         if os.path.basename(env) == project_env_name][0]

examples/skein-project/setup.py

@@ -4,7 +4,7 @@
     name='skein_project',
     version='0.0.1',
     packages=setuptools.find_packages(),
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     install_requires=[
         "skein",
         "numpy",

examples/skein-project/skein_example.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # install venv
-python3.6 -m venv skein_project_env
+python3.9 -m venv skein_project_env
 . skein_project_env/bin/activate
 pip install --upgrade pip setuptools
 pip install -e .

examples/spark-with-S3/README.md

@@ -13,7 +13,7 @@ docker build -t spark-docker ./examples/spark-with-S3
 2) Start standalone Spark & local S3 containers (using [minio](https://min.io/))
 
 ```bash
-docker-compose -f ./examples/spark-with-S3/docker-compose.yml up -d
+docker-compose -f ./examples/spark-with-S3/docker compose.yml up -d
 ```
 
 ### Quick run

examples/spark-with-S3/scripts/run_spark_example.sh

@@ -5,10 +5,6 @@ $1 -m venv /tmp/pyspark_env
 pip install -U pip setuptools wheel
 pip install pypandoc<1.8
 pip install s3fs pandas pyspark==$2
-if [ $1 == "python3.6" ]
-then
-   pip install 'pandas<1.0.0' pyarrow==0.14.1
-fi
 pip install -e /cluster-pack
 
 curr_dir=$(dirname "$0")

requirements.txt

@@ -1,7 +1,7 @@
 cloudpickle
 pex==2.1.137
 conda-pack
-pip>=18.1
+pip>=22.0
 pyarrow<16.0.0
 fire
 types-setuptools

scripts/hadoop_hdfs_test.ps1

@@ -3,7 +3,7 @@
 # run from root dir cluster-pack scripts/hadoop_hdfs_tests.ps1
 pip install hadoop-test-cluster
 htcluster startup --image cdh5 --mount .:cluster-pack
-# to avoid sharing files on the worker node we copy the python3.6 install script via hdfs to worker /tmp folder
+# to avoid sharing files on the worker node we copy the python install script via hdfs to worker /tmp folder
 htcluster exec -u root -s edge -- chown -R testuser /home/testuser
 htcluster exec -u root -s edge -- /home/testuser/cluster-pack/tests/integration/install_python.sh
 htcluster exec -u root -s edge -- hdfs dfs -put /home/testuser/cluster-pack/tests/integration/install_python.sh hdfs:///tmp

setup.py

@@ -30,10 +30,11 @@ def _read_reqs(relpath):
     "Environment :: Console",
     "License :: OSI Approved :: Apache Software License",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Topic :: Software Development :: Libraries"
 ]
 
@@ -44,7 +45,7 @@ def _read_reqs(relpath):
     version=versioneer.get_version(),
     install_requires=REQUIREMENTS,
     tests_require=["pytest"],
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     maintainer="Criteo",
     maintainer_email="[email protected]",
     description=DESCRIPTION,

tests-requirements.txt

@@ -5,4 +5,4 @@ pytest
 pyflakes==2.4.0
 pylama
 mypy
-protobuf!=4.21.0
+numpy<2  # breaking changes from numpy 2+, can be removed when all dependencies are compatible

tests/integration/hadoop_hdfs_tests.sh

@@ -4,7 +4,7 @@ set -e
 
 pushd cluster-pack
     rm -rf ~/venv
-    python3.6 -m venv ~/venv
+    python3.9 -m venv ~/venv
     source ~/venv/bin/activate
     pip install -U wheel pip setuptools
     pip install -e .

tests/integration/install_python.sh

@@ -2,18 +2,18 @@
 
 set -e
 
-if [ -d "/usr/src/Python-3.6.10" ] 
+if [ -d "/usr/src/Python-3.9.15" ]
 then
-	echo "Python already installed in /usr/src/Python-3.6.10"
+	echo "Python already installed in /usr/src/Python-3.9.15"
 	exit
 fi
 
 # install python interpreter globally
 yum install -y wget gcc openssl-devel bzip2-devel libffi-devel
 pushd /usr/src
-wget https://www.python.org/ftp/python/3.6.10/Python-3.6.10.tgz
-tar xzf Python-3.6.10.tgz
-pushd Python-3.6.10
+wget https://www.python.org/ftp/python/3.9.15/Python-3.9.15.tgz
+tar xzf Python-3.9.15.tgz
+pushd Python-3.9.15
   ./configure --enable-optimizations
   make altinstall
-rm /usr/src/Python-3.6.10.tgz
+rm /usr/src/Python-3.9.15.tgz

tests/resources/conda.yaml

@@ -4,5 +4,5 @@ name: tutorial
 channels:
   - defaults
 dependencies:
-  - python=3.6
+  - python=3.9
   - botocore==1.17.12

tests/test_packaging.py

@@ -18,6 +18,7 @@
 MYARCHIVE_FILENAME = "myarchive.pex"
 MYARCHIVE_METADATA = "myarchive.json"
 VARNAME = 'VARNAME'
+PINNED_VERSIONS_FOR_COMPATIBILITY_ISSUE = {"numpy": "numpy<2"}
 
 
 def test_get_virtualenv_name():
@@ -37,7 +38,7 @@ def test_get_empty_editable_requirements():
     with tempfile.TemporaryDirectory() as tempdir:
         _create_venv(tempdir)
         subprocess.check_call([f"{tempdir}/bin/python", "-m", "pip", "install",
-                               "cloudpickle", _get_editable_package_name(), "pip==18.1"])
+                               "cloudpickle", _get_editable_package_name(), "pip==22.0"])
         editable_requirements = packaging._get_editable_requirements(f"{tempdir}/bin/python")
         assert len(editable_requirements) == 0
 
@@ -46,7 +47,7 @@ def test_get_empty_non_editable_requirements():
     with tempfile.TemporaryDirectory() as tempdir:
         _create_venv(tempdir)
         subprocess.check_call([f"{tempdir}/bin/python", "-m", "pip", "install",
-                               "-e", _get_editable_package_name(), "pip==18.1"])
+                               "-e", _get_editable_package_name(), "pip==22.0"])
         non_editable_requirements = packaging.get_non_editable_requirements(
             f"{tempdir}/bin/python")
         assert len(non_editable_requirements) == 2
@@ -75,7 +76,6 @@ def test__get_editable_requirements_for_src_layout():
         assert "user_lib2" in pkg_names
 
 
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher")
 def test__get_editable_requirements_withpip23():
     with tempfile.TemporaryDirectory() as tempdir:
         _create_venv(tempdir)
@@ -101,7 +101,7 @@ def _create_venv(tempdir: str):
     subprocess.check_call([sys.executable, "-m", "venv", f"{tempdir}"])
 
 
-def _pip_install(tempdir: str, pip_version: str = "18.1", use_src_layout: bool = False):
+def _pip_install(tempdir: str, pip_version: str = "22.0", use_src_layout: bool = False):
     subprocess.check_call([f"{tempdir}/bin/python", "-m", "pip", "install",
                            "cloudpickle", f"pip=={pip_version}"])
     pkg = (_get_editable_package_name_src_layout() if use_src_layout
@@ -123,7 +123,7 @@ def test_get_current_pex_filepath():
     with tempfile.TemporaryDirectory() as tempdir:
         path_to_pex = f"{tempdir}/out.pex"
         packaging.pack_in_pex(
-            ["numpy"],
+            [PINNED_VERSIONS_FOR_COMPATIBILITY_ISSUE['numpy']],
             path_to_pex,
             # make isolated pex from current pytest virtual env
             pex_inherit_path="false")
@@ -215,7 +215,7 @@ def test_pack_in_pex(pyarrow_version, expectation):
 
 def test_pack_in_pex_with_allow_large():
     with tempfile.TemporaryDirectory() as tempdir:
-        requirements = ["pyarrow==6.0.1"]
+        requirements = [PINNED_VERSIONS_FOR_COMPATIBILITY_ISSUE['numpy'], "pyarrow==6.0.1"]
         packaging.pack_in_pex(
             requirements,
             f"{tempdir}/out.pex",
@@ -241,7 +241,7 @@ def test_pack_in_pex_with_allow_large():
 
 def test_pack_in_pex_with_include_tools():
     with tempfile.TemporaryDirectory() as tempdir:
-        requirements = ["pyarrow==6.0.1"]
+        requirements = [PINNED_VERSIONS_FOR_COMPATIBILITY_ISSUE['numpy'], "pyarrow==6.0.1"]
         packaging.pack_in_pex(
             requirements,
             f"{tempdir}/out.pex",
@@ -310,12 +310,8 @@ def test_pack_in_pex_with_large_correctly_retrieves_zip_archive(is_large_pex, pa
 
 
 def test_pack_in_pex_with_additional_repo():
-    if sys.version_info.minor == 6:
-        # dependency issue with available pytorch on https://download.pytorch.org/whl/cpu
-        return
-
     with tempfile.TemporaryDirectory() as tempdir:
-        requirements = ["setuptools", "torch",
+        requirements = ["torch",
                         "typing-extensions<=3.7.4.3; python_version<'3.8'",
                         "networkx<2.6; python_version<'3.9'"]
         packaging.pack_in_pex(

tests/test_uploader.py

@@ -1,3 +1,5 @@
+import re
+import subprocess
 import sys
 import contextlib
 import json
@@ -77,11 +79,6 @@ def test_update_no_metadata():
                   'platform': 'fake_platform2',
                   'python_version': '3.9.10'},
                  False),
-    pytest.param(["a==2.0", "b==1.0"],
-                 {'package_installed': ["a==2.0", "b==1.0"],
-                  'platform': 'fake_platform',
-                  'python_version': '3.6.8'},
-                 False),
 ])
 @mock.patch(f'{MODULE_TO_TEST}.platform.platform')
 @mock.patch(f'{MODULE_TO_TEST}.sys')
@@ -446,6 +443,21 @@ def test__unique_filename(spec_file, expected):
     assert expected == uploader._unique_filename(spec_file, packaging.PEX_PACKER)
 
 
+def get_latest_pip_version() -> str:
+    p = subprocess.Popen("pip index versions pip", shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    for line in p.stdout.readlines():
+        result = re.match(r"pip \((.+)\)", line.decode("utf-8"))
+        if result:
+            return result.group(1)
+    return None
+
+
+def test_latest_pip():
+
+    assert get_latest_pip_version() is not None
+
+
 def test_format_pex_requirements():
     with tempfile.TemporaryDirectory() as tempdir:
         requirements = ["pipdeptree==2.0.0", "six==1.15.0"]
@@ -456,8 +468,8 @@ def test_format_pex_requirements():
             pex_inherit_path="false")
         pex_info = PexInfo.from_pex(f"{tempdir}/out.pex")
         cleaned_requirements = uploader._format_pex_requirements(pex_info)
-        pip_version = 'pip==21.3.1' if sys.version_info.minor == 6 else 'pip==24.0'
-        assert [pip_version, 'pipdeptree==2.0.0', 'six==1.15.0'] == cleaned_requirements
+        assert [f'pip=={get_latest_pip_version()}',
+                'pipdeptree==2.0.0', 'six==1.15.0'] == cleaned_requirements
 
 
 @pytest.mark.parametrize("req, expected", [