Merge branch 'main' into optimizer_improvements

dask-contrib · Jan 9, 2024 · 3f3d401 · 3f3d401
2 parents 2caaa9b + 3ffba21
commit 3f3d401
Show file tree

Hide file tree

Showing 42 changed files with 193 additions and 180 deletions.
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         arch: ["linux-64", "linux-aarch64"]
     steps:
       - name: Manage disk space
@@ -72,7 +72,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Install dependencies
         run: |

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -11,7 +11,7 @@ on:
       - Cargo.toml
       - Cargo.lock
       - pyproject.toml
-      - docker/**
+      - continuous_integration/docker/**
       - .github/workflows/docker.yml
 
 # When this workflow is queued, automatically cancel any previous running
@@ -47,7 +47,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/main.dockerfile
+          file: ./continuous_integration/docker/main.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
           platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_main.outputs.tags }}
@@ -68,7 +68,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/cloud.dockerfile
+          file: ./continuous_integration/docker/cloud.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
           platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_cloud.outputs.tags }}

diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml
@@ -36,21 +36,21 @@ jobs:
     name: "Test upstream dev (${{ matrix.os }}, python: ${{ matrix.python }}, distributed: ${{ matrix.distributed }})"
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
       - uses: actions/checkout@v4
@@ -75,7 +75,6 @@ jobs:
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
-          mamba install -c conda-forge "sasl>=0.3.1"
           docker pull bde2020/hive:2.3.2-postgresql-metastore
           docker pull bde2020/hive-metastore-postgresql:2.3.0
       - name: Install upstream dev Dask
@@ -109,8 +108,6 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          # TODO: drop support for python 3.8, add support for python 3.11
-          # https://github.com/dask-contrib/dask-sql/pull/1143
           python-version: "3.9"
           channel-priority: strict
       - name: Optionally update upstream cargo dependencies

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -37,21 +37,21 @@ jobs:
     needs: [detect-ci-trigger]
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
       - uses: actions/checkout@v4
@@ -76,7 +76,6 @@ jobs:
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
-          mamba install -c conda-forge "sasl>=0.3.1"
           docker pull bde2020/hive:2.3.2-postgresql-metastore
           docker pull bde2020/hive-metastore-postgresql:2.3.0
       - name: Optionally install upstream dev Dask
@@ -107,9 +106,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          # TODO: drop support for python 3.8, add support for python 3.11
-          # https://github.com/dask-contrib/dask-sql/pull/1143
-          python-version: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && '3.9' || '3.8' }}
+          python-version: "3.9"
           channel-priority: strict
       - name: Install dependencies and nothing else
         run: |

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -19,7 +19,7 @@ rustup update
 To initialize and activate the conda environment for a given Python version:
 
 ```
-conda env create -f dask-sql/continuous_integration/environment-{$PYTHON_VER}-dev.yaml
+conda env create -f dask-sql/continuous_integration/environment-{$PYTHON_VER}.yaml
 conda activate dask-sql
 ```
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -10,11 +10,11 @@ rust-version = "1.72"
 include = ["/src", "/dask_sql", "/LICENSE.txt", "pyproject.toml", "Cargo.toml", "Cargo.lock"]
 
 [dependencies]
-async-trait = "0.1.74"
+async-trait = "0.1.77"
 datafusion-python = { git = "https://github.com/apache/arrow-datafusion-python.git", ref = "da6c183" }
 env_logger = "0.10"
 log = "^0.4"
-pyo3 = { version = "0.19.2", features = ["extension-module", "abi3", "abi3-py38"] }
+pyo3 = { version = "0.19.2", features = ["extension-module", "abi3", "abi3-py39"] }
 pyo3-log = "0.9.0"
 
 [build-dependencies]

diff --git a/README.md b/README.md
@@ -101,7 +101,7 @@ If you want to have the newest (unreleased) `dask-sql` version or if you plan to
 
 Create a new conda environment and install the development environment:
 
-    conda env create -f continuous_integration/environment-3.9-dev.yaml
+    conda env create -f continuous_integration/environment-3.9.yaml
 
 It is not recommended to use `pip` instead of `conda` for the environment setup.
 

diff --git a/docker/cloud.dockerfile → ...nuous_integration/docker/cloud.dockerfile b/docker/cloud.dockerfile → ...nuous_integration/docker/cloud.dockerfile
diff --git a/docker/conda.txt → continuous_integration/docker/conda.txt b/docker/conda.txt → continuous_integration/docker/conda.txt
@@ -1,5 +1,5 @@
-python>=3.8
-dask>=2022.3.0,<=2023.11.0
+python>=3.9
+dask>=2022.3.0
 pandas>=1.4.0
 jpype1>=1.0.2
 openjdk>=8

diff --git a/docker/main.dockerfile → ...inuous_integration/docker/main.dockerfile b/docker/main.dockerfile → ...inuous_integration/docker/main.dockerfile
@@ -11,12 +11,12 @@ RUN sh /rustup-init.sh -y --default-toolchain=stable --profile=minimal \
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 # Install conda dependencies for dask-sql
-COPY docker/conda.txt /opt/dask_sql/
+COPY continuous_integration/docker/conda.txt /opt/dask_sql/
 RUN mamba install -y \
     # build requirements
     "maturin>=1.3,<1.4" \
     # core dependencies
-    "dask>=2022.3.0,<=2023.11.0" \
+    "dask>=2022.3.0" \
     "pandas>=1.4.0" \
     "fastapi>=0.92.0" \
     "httpx>=0.24.1" \
@@ -44,7 +44,7 @@ RUN cd /opt/dask_sql/ \
     && CONDA_PREFIX="/opt/conda/" maturin develop
 
 # Set the script to execute
-COPY scripts/startup_script.py /opt/dask_sql/startup_script.py
+COPY continuous_integration/scripts/startup_script.py /opt/dask_sql/startup_script.py
 
 EXPOSE 8080
 ENTRYPOINT [ "/usr/bin/prepare.sh", "/opt/conda/bin/python", "/opt/dask_sql/startup_script.py" ]
diff --git a/...ous_integration/environment-3.10-dev.yaml → continuous_integration/environment-3.10.yaml b/...ous_integration/environment-3.10-dev.yaml → continuous_integration/environment-3.10.yaml
@@ -1,20 +1,17 @@
 name: dask-sql
 channels:
 - conda-forge
-- nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0,<=2023.11.0
+- dask>=2022.3.0
 - fastapi>=0.92.0
 - fugue>=0.7.3
 - httpx>=0.24.1
 - intake>=0.6.0
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-# FIXME: mlflow 2.6.0 has import issues related to pydantic
-# https://github.com/mlflow/mlflow/issues/9331
-- mlflow<2.6
+- mlflow>=2.0
 - mock
 - numpy>=1.21.6
 - pandas>=1.4.0