Drop python 3.8, add 3.11 & 3.12 to testing (#1143)

* Initial python version bump in CI * Add libprotobuf to GPU CI environments * Replace mentions of old env files * Remove strict channel priority to try to unblock env solves? * Establish minimum version for mlflow * Revert "Remove strict channel priority to try to unblock env solves?" This reverts commit e454833. * Try strict channel priority without nodefaults * Bump mlflow min version to fix windows failures * Build python 3.11 wheels * Run wheel builds in PR test * Try protoc action in wheels build to unblock * Skip hive testing on 3.11 for now * Fix workflow syntax errors * Stop running wheel CI * Bump pyo3 abi minor version * Initial run of pyupgrade to py39 * Continue marking test_dask_fsql as flaky * More places to drop 3.8 * Try running tests on python 3.12 * Add environment file * Skip sasl installation * Drop protoc build dep * Drop mlflow constraint * Set min version for mlflow * Drop mlflow from 3.12 tests for now * Relocate docker/server files to continuous_integration * Unpin dask/distributed * unpin 3.9 gpu environment * add 3.12 to classifiers * unpin dask in gpuci 3.9
dask-contrib · Jan 9, 2024 · ce51df3 · ce51df3
1 parent dbb36a4
commit ce51df3
Show file tree

Hide file tree

Showing 41 changed files with 172 additions and 159 deletions.
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         arch: ["linux-64", "linux-aarch64"]
     steps:
       - name: Manage disk space
@@ -72,7 +72,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Install dependencies
         run: |

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -11,7 +11,7 @@ on:
       - Cargo.toml
       - Cargo.lock
       - pyproject.toml
-      - docker/**
+      - continuous_integration/docker/**
       - .github/workflows/docker.yml
 
 # When this workflow is queued, automatically cancel any previous running
@@ -47,7 +47,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/main.dockerfile
+          file: ./continuous_integration/docker/main.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
           platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_main.outputs.tags }}
@@ -68,7 +68,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/cloud.dockerfile
+          file: ./continuous_integration/docker/cloud.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
           platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_cloud.outputs.tags }}

diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml
@@ -36,21 +36,21 @@ jobs:
     name: "Test upstream dev (${{ matrix.os }}, python: ${{ matrix.python }}, distributed: ${{ matrix.distributed }})"
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
       - uses: actions/checkout@v4
@@ -75,7 +75,6 @@ jobs:
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
-          mamba install -c conda-forge "sasl>=0.3.1"
           docker pull bde2020/hive:2.3.2-postgresql-metastore
           docker pull bde2020/hive-metastore-postgresql:2.3.0
       - name: Install upstream dev Dask
@@ -109,8 +108,6 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          # TODO: drop support for python 3.8, add support for python 3.11
-          # https://github.com/dask-contrib/dask-sql/pull/1143
           python-version: "3.9"
           channel-priority: strict
       - name: Optionally update upstream cargo dependencies

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -37,21 +37,21 @@ jobs:
     needs: [detect-ci-trigger]
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
       - uses: actions/checkout@v4
@@ -76,7 +76,6 @@ jobs:
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
-          mamba install -c conda-forge "sasl>=0.3.1"
           docker pull bde2020/hive:2.3.2-postgresql-metastore
           docker pull bde2020/hive-metastore-postgresql:2.3.0
       - name: Optionally install upstream dev Dask
@@ -107,9 +106,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          # TODO: drop support for python 3.8, add support for python 3.11
-          # https://github.com/dask-contrib/dask-sql/pull/1143
-          python-version: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && '3.9' || '3.8' }}
+          python-version: "3.9"
           channel-priority: strict
       - name: Install dependencies and nothing else
         run: |

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -19,7 +19,7 @@ rustup update
 To initialize and activate the conda environment for a given Python version:
 
 ```
-conda env create -f dask-sql/continuous_integration/environment-{$PYTHON_VER}-dev.yaml
+conda env create -f dask-sql/continuous_integration/environment-{$PYTHON_VER}.yaml
 conda activate dask-sql
 ```
 

diff --git a/Cargo.toml b/Cargo.toml
@@ -14,7 +14,7 @@ async-trait = "0.1.74"
 datafusion-python = { git = "https://github.com/apache/arrow-datafusion-python.git", ref = "da6c183" }
 env_logger = "0.10"
 log = "^0.4"
-pyo3 = { version = "0.19.2", features = ["extension-module", "abi3", "abi3-py38"] }
+pyo3 = { version = "0.19.2", features = ["extension-module", "abi3", "abi3-py39"] }
 pyo3-log = "0.9.0"
 
 [build-dependencies]

diff --git a/README.md b/README.md
@@ -101,7 +101,7 @@ If you want to have the newest (unreleased) `dask-sql` version or if you plan to
 
 Create a new conda environment and install the development environment:
 
-    conda env create -f continuous_integration/environment-3.9-dev.yaml
+    conda env create -f continuous_integration/environment-3.9.yaml
 
 It is not recommended to use `pip` instead of `conda` for the environment setup.
 

diff --git a/docker/cloud.dockerfile → ...nuous_integration/docker/cloud.dockerfile b/docker/cloud.dockerfile → ...nuous_integration/docker/cloud.dockerfile
diff --git a/docker/conda.txt → continuous_integration/docker/conda.txt b/docker/conda.txt → continuous_integration/docker/conda.txt
@@ -1,5 +1,5 @@
-python>=3.8
-dask>=2022.3.0,<=2023.11.0
+python>=3.9
+dask>=2022.3.0
 pandas>=1.4.0
 jpype1>=1.0.2
 openjdk>=8

diff --git a/docker/main.dockerfile → ...inuous_integration/docker/main.dockerfile b/docker/main.dockerfile → ...inuous_integration/docker/main.dockerfile
@@ -11,12 +11,12 @@ RUN sh /rustup-init.sh -y --default-toolchain=stable --profile=minimal \
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 # Install conda dependencies for dask-sql
-COPY docker/conda.txt /opt/dask_sql/
+COPY continuous_integration/docker/conda.txt /opt/dask_sql/
 RUN mamba install -y \
     # build requirements
     "maturin>=1.3,<1.4" \
     # core dependencies
-    "dask>=2022.3.0,<=2023.11.0" \
+    "dask>=2022.3.0" \
     "pandas>=1.4.0" \
     "fastapi>=0.92.0" \
     "httpx>=0.24.1" \
@@ -44,7 +44,7 @@ RUN cd /opt/dask_sql/ \
     && CONDA_PREFIX="/opt/conda/" maturin develop
 
 # Set the script to execute
-COPY scripts/startup_script.py /opt/dask_sql/startup_script.py
+COPY continuous_integration/scripts/startup_script.py /opt/dask_sql/startup_script.py
 
 EXPOSE 8080
 ENTRYPOINT [ "/usr/bin/prepare.sh", "/opt/conda/bin/python", "/opt/dask_sql/startup_script.py" ]
diff --git a/...ous_integration/environment-3.10-dev.yaml → continuous_integration/environment-3.10.yaml b/...ous_integration/environment-3.10-dev.yaml → continuous_integration/environment-3.10.yaml
@@ -1,20 +1,17 @@
 name: dask-sql
 channels:
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0,<=2023.11.0
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1
 - intake>=0.6.0
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-# FIXME: mlflow 2.6.0 has import issues related to pydantic
-# https://github.com/mlflow/mlflow/issues/9331
-- mlflow<2.6
+- mlflow>=2.0
 - mock
 - numpy>=1.21.6
 - pandas>=1.4.0

diff --git a/...uous_integration/environment-3.9-dev.yaml → continuous_integration/environment-3.11.yaml b/...uous_integration/environment-3.9-dev.yaml → continuous_integration/environment-3.11.yaml
@@ -1,20 +1,17 @@
 name: dask-sql
 channels:
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0,<=2023.11.0
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1
 - intake>=0.6.0
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-# FIXME: mlflow 2.6.0 has import issues related to pydantic
-# https://github.com/mlflow/mlflow/issues/9331
-- mlflow<2.6
+- mlflow>=2.0
 - mock
 - numpy>=1.21.6
 - pandas>=1.4.0
@@ -28,7 +25,7 @@ dependencies:
 - pytest-rerunfailures
 - pytest-xdist
 - pytest
-- python=3.9
+- python=3.11
 - scikit-learn>=1.0.0
 - sphinx
 - sqlalchemy<2

diff --git a/continuous_integration/environment-3.12.yaml b/continuous_integration/environment-3.12.yaml
@@ -0,0 +1,38 @@
+name: dask-sql
+channels:
+- conda-forge
+dependencies:
+- c-compiler
+- dask>=2022.3.0
+- fastapi>=0.92.0
+- fugue>=0.7.3
+- httpx>=0.24.1
+- intake>=0.6.0
+- jsonschema
+- lightgbm
+- maturin>=1.3,<1.4
+# TODO: add once mlflow 3.12 builds are available
+# - mlflow>=2.0
+- mock
+- numpy>=1.21.6
+- pandas>=1.4.0
+- pre-commit
+- prompt_toolkit>=3.0.8
+- psycopg2
+- pyarrow>=6.0.2
+- pygments>=2.7.1
+- pyhive
+- pytest-cov
+- pytest-rerunfailures
+- pytest-xdist
+- pytest
+- python=3.12
+- scikit-learn>=1.0.0
+- sphinx
+- sqlalchemy<2
+- tpot>=0.12.0
+# FIXME: https://github.com/fugue-project/fugue/issues/526
+- triad<0.9.2
+- tzlocal>=2.1
+- uvicorn>=0.13.4
+- zlib
diff --git a/...uous_integration/environment-3.8-dev.yaml → continuous_integration/environment-3.9.yaml b/...uous_integration/environment-3.8-dev.yaml → continuous_integration/environment-3.9.yaml
@@ -1,7 +1,6 @@
-name: dask-sql
+name: dask-sql-py39
 channels:
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
 - dask=2022.3.0
@@ -11,10 +10,8 @@ dependencies:
 - intake=0.6.0
 - jsonschema
 - lightgbm
-- maturin>=1.3,<1.4
-# FIXME: mlflow 2.6.0 has import issues related to pydantic
-# https://github.com/mlflow/mlflow/issues/9331
-- mlflow<2.6
+- maturin=1.3
+- mlflow=2.0
 - mock
 - numpy=1.21.6
 - pandas=1.4.0
@@ -28,7 +25,7 @@ dependencies:
 - pytest-rerunfailures
 - pytest-xdist
 - pytest
-- python=3.8
+- python=3.9
 - scikit-learn=1.0.0
 - sphinx
 - sqlalchemy<2

diff --git a/continuous_integration/gpuci/environment-3.10.yaml b/continuous_integration/gpuci/environment-3.10.yaml
@@ -8,17 +8,15 @@ channels:
 dependencies:
 - c-compiler
 - zlib
-- dask>=2022.3.0,<=2023.11.0
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1
 - intake>=0.6.0
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-# FIXME: mlflow 2.6.0 has import issues related to pydantic
-# https://github.com/mlflow/mlflow/issues/9331
-- mlflow<2.6
+- mlflow>=2.0
 - mock
 - numpy>=1.21.6
 - pandas>=1.4.0
@@ -51,8 +49,7 @@ dependencies:
 - ucx-py=0.36
 - xgboost=*=rapidsai_py*
 - libxgboost=*=rapidsai_h*
-# TODO: unpin after RAPIDS 24.02 release
-# - pip
-# - pip:
-#     - git+https://github.com/dask/dask
-#     - git+https://github.com/dask/distributed
+- pip
+- pip:
+    - git+https://github.com/dask/dask
+    - git+https://github.com/dask/distributed
diff --git a/continuous_integration/gpuci/environment-3.9.yaml b/continuous_integration/gpuci/environment-3.9.yaml
@@ -8,17 +8,15 @@ channels:
 dependencies:
 - c-compiler
 - zlib
-- dask>=2022.3.0,<=2023.11.0
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1
 - intake>=0.6.0
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-# FIXME: mlflow 2.6.0 has import issues related to pydantic
-# https://github.com/mlflow/mlflow/issues/9331
-- mlflow<2.6
+- mlflow>=2.0
 - mock
 - numpy>=1.21.6
 - pandas>=1.4.0
@@ -51,8 +49,7 @@ dependencies:
 - ucx-py=0.36
 - xgboost=*=rapidsai_py*
 - libxgboost=*=rapidsai_h*
-# TODO: unpin after RAPIDS 24.02 release
-# - pip
-# - pip:
-#     - git+https://github.com/dask/dask
-#     - git+https://github.com/dask/distributed
+- pip
+- pip:
+    - git+https://github.com/dask/dask
+    - git+https://github.com/dask/distributed
diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml
@@ -32,7 +32,7 @@ requirements:
     - xz  # [linux64]
   run:
     - python
-    - dask >=2022.3.0,<=2023.11.0
+    - dask >=2022.3.0
     - pandas >=1.4.0
     - fastapi >=0.92.0
     - httpx >=0.24.1

diff --git a/scripts/startup_script.py → ...ous_integration/scripts/startup_script.py b/scripts/startup_script.py → ...ous_integration/scripts/startup_script.py