Merge remote-tracking branch 'upstream/main'

jdye64 · Jun 14, 2023 · 28fce59 · 28fce59
2 parents 4b86547 + f1fd764
commit 28fce59
Show file tree

Hide file tree

Showing 26 changed files with 470 additions and 118 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -18,6 +18,5 @@ updates:
       # Check for updates to GitHub Actions every weekday
       interval: "weekly"
     ignore:
-      # ignore cibw patch updates
+      # prefer updating cibuildwheel manually as needed
       - dependency-name: "pypa/cibuildwheel"
-        update-types: ["version-update:semver-patch"]
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -3,7 +3,6 @@ on:
   push:
     branches:
       - main
-      - datafusion-sql-planner
   pull_request:
     paths:
       - setup.py
@@ -29,8 +28,13 @@ defaults:
 
 jobs:
   conda:
-    name: Build (and optionally upload) the conda nightly
+    name: "Build conda nightlies (python: ${{ matrix.python }}, arch: ${{ matrix.arch }})"
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.8", "3.9", "3.10"]
+        arch: ["linux-64", "linux-aarch64"]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -49,23 +53,35 @@ jobs:
           which python
           pip list
           mamba list
-      - name: Build conda package
+      - name: Build conda packages
         run: |
           # suffix for nightly package versions
           export VERSION_SUFFIX=a`date +%y%m%d`
 
           conda mambabuild continuous_integration/recipe \
+                           --python ${{ matrix.python }} \
+                           --variants "{target_platform: [${{ matrix.arch }}]}" \
+                           --no-test \
                            --no-anaconda-upload \
-                           --output-folder .
-      - name: Upload conda package
+                           --output-folder packages
+      - name: Test conda packages
+        if: matrix.arch == 'linux-64'  # can only test native platform packages
+        run: |
+          conda mambabuild --test packages/${{ matrix.arch }}/*.tar.bz2
+      - name: Upload conda packages as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: "conda nightlies (python - ${{ matrix.python }}, arch - ${{ matrix.arch }})"
+          # need to install all conda channel metadata to properly install locally
+          path: packages/
+      - name: Upload conda packages to Anaconda
         if: |
           github.event_name == 'push'
           && github.repository == 'dask-contrib/dask-sql'
         env:
           ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
-          LABEL: ${{ github.ref == 'refs/heads/datafusion-sql-planner' && 'dev_datafusion' || 'dev' }}
         run: |
           # install anaconda for upload
           mamba install -c conda-forge anaconda-client
 
-          anaconda upload --label $LABEL linux-64/*.tar.bz2
+          anaconda upload --label dev packages/${{ matrix.arch }}/*.tar.bz2
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -2,6 +2,18 @@ name: Upload Python package
 on:
   release:
     types: [created]
+  pull_request:
+    paths:
+      - .github/workflows/release.yml
+
+# When this workflow is queued, automatically cancel any previous running
+# or pending jobs from the same branch
+concurrency:
+  group: release-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  upload: ${{ github.event_name == 'release' && github.repository == 'dask-contrib/dask-sql' }}
 
 # Required shell entrypoint to have properly activated conda environments
 defaults:
@@ -21,13 +33,19 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up QEMU for linux aarch64
-        if: contains(matrix.os, 'ubuntu')
+      - name: Install Protoc
+        if: matrix.os != 'ubuntu-latest'
+        uses: arduino/setup-protoc@v1
+        with:
+          version: '3.x'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU for linux-aarch64
+        if: matrix.os == 'ubuntu-latest'
         uses: docker/setup-qemu-action@v2
         with:
           platforms: arm64
-      - name: Add arm64 target for macos
-        if: contains(matrix.os, 'macos')
+      - name: Add rust toolchain target for macos-aarch64
+        if: matrix.os == 'macos-latest'
         run: rustup target add aarch64-apple-darwin
       - name: Build wheels
         uses: pypa/[email protected]
@@ -37,13 +55,21 @@ jobs:
           CIBW_ARCHS_LINUX: 'aarch64 x86_64'
           CIBW_ARCHS_WINDOWS: 'AMD64'
           CIBW_ARCHS_MACOS: 'x86_64 arm64'
-          CIBW_ENVIRONMENT_LINUX: 'CARGO_NET_GIT_FETCH_WITH_CLI="true" PATH="$HOME/.cargo/bin:$PATH"'
           # Without CARGO_NET_GIT_FETCH_WITH_CLI we oom (https://github.com/rust-lang/cargo/issues/10583)
+          CIBW_ENVIRONMENT_LINUX: >
+            CARGO_NET_GIT_FETCH_WITH_CLI="true"
+            PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH"
           CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"'
           CIBW_BEFORE_BUILD: 'pip install -U setuptools-rust'
           CIBW_BEFORE_BUILD_LINUX: >
+            ARCH=$([ $(uname -m) == x86_64 ] && echo x86_64 || echo aarch_64) &&
+            DOWNLOAD_URL=$(curl --retry 6 --retry-delay 10 -s https://api.github.com/repos/protocolbuffers/protobuf/releases/latest | grep -o '"browser_download_url": "[^"]*' | cut -d'"' -f4 | grep "\linux-${ARCH}.zip$") &&
+            curl --retry 6 --retry-delay 10 -LO $DOWNLOAD_URL &&
+            unzip protoc-*-linux-$ARCH.zip -d $HOME/.local &&
+            protoc --version &&
             pip install -U setuptools-rust &&
-            curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=minimal -y &&
+            pip list &&
+            curl --retry 6 --retry-delay 10 https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=minimal -y &&
             rustup show
         with:
           package-dir: .
@@ -56,6 +82,18 @@ jobs:
           use-mamba: true
           python-version: "3.8"
           channel-priority: strict
+      # FIXME: aarch64 builds require glibc>=2.32 to prevent TLS allocation error when importing;
+      # is there any way we could modify the TLS model of its shared object to avoid this?
+      # https://bugzilla.redhat.com/show_bug.cgi?id=1722181
+      # xref: https://github.com/dask-contrib/dask-sql/issues/1169
+      - name: Update platform tag for aarch64 wheel
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          mamba install wheel
+
+          wheel tags --platform-tag=manylinux_2_32_aarch64 \
+                     --remove \
+                     dist/*_aarch64.whl
       - name: Check dist files
         run: |
           mamba install twine
@@ -68,6 +106,7 @@ jobs:
           name: wheels for py3.${{ matrix.python }} on ${{ matrix.os }}
           path: dist/*
       - name: Publish package
+        if: env.upload == 'true'
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
@@ -96,6 +135,7 @@ jobs:
           twine check dist/*
           ls -lh dist/
       - name: Publish source distribution
+        if: env.upload == 'true'
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

diff --git a/continuous_integration/environment-3.10-dev.yaml b/continuous_integration/environment-3.10-dev.yaml
@@ -4,7 +4,7 @@ channels:
 - nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0
+- dask>=2022.3.0,<=2023.5.1
 # FIXME: handling is needed for httpx-based fastapi>=0.87.0
 - fastapi>=0.69.0,<0.87.0
 - fugue>=0.7.3
@@ -14,9 +14,7 @@ dependencies:
 - maturin>=0.12.8
 - mlflow
 - mock
-# tpot imports fail with numpy >=1.24.0
-# https://github.com/EpistasisLab/tpot/issues/1281
-- numpy<1.24.0
+- numpy>=1.21.6
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
@@ -33,7 +31,7 @@ dependencies:
 - setuptools-rust>=1.5.2
 - sphinx
 - sqlalchemy<2
-- tpot
+- tpot>=0.12.0
 - tzlocal>=2.1
 - uvicorn>=0.13.4
 - libprotobuf=3
diff --git a/continuous_integration/environment-3.8-dev.yaml b/continuous_integration/environment-3.8-dev.yaml
@@ -13,9 +13,7 @@ dependencies:
 - maturin=0.12.8
 - mlflow
 - mock
-# tpot imports fail with numpy >=1.24.0
-# https://github.com/EpistasisLab/tpot/issues/1281
-- numpy<1.24.0
+- numpy=1.21.6
 - pandas=1.4.0
 - pre-commit
 - prompt_toolkit=3.0.8
@@ -32,7 +30,7 @@ dependencies:
 - setuptools-rust=1.5.2
 - sphinx
 - sqlalchemy<2
-- tpot
+- tpot>=0.12.0
 - tzlocal=2.1
 - uvicorn=0.13.4
 - libprotobuf=3
diff --git a/continuous_integration/environment-3.9-dev.yaml b/continuous_integration/environment-3.9-dev.yaml
@@ -4,7 +4,7 @@ channels:
 - nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0
+- dask>=2022.3.0,<=2023.5.1
 # FIXME: handling is needed for httpx-based fastapi>=0.87.0
 - fastapi>=0.69.0,<0.87.0
 - fugue>=0.7.3
@@ -14,9 +14,7 @@ dependencies:
 - maturin>=0.12.8
 - mlflow
 - mock
-# tpot imports fail with numpy >=1.24.0
-# https://github.com/EpistasisLab/tpot/issues/1281
-- numpy<1.24.0
+- numpy>=1.21.6
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
@@ -33,7 +31,7 @@ dependencies:
 - setuptools-rust>=1.5.2
 - sphinx
 - sqlalchemy<2
-- tpot
+- tpot>=0.12.0
 - tzlocal>=2.1
 - uvicorn>=0.13.4
 - libprotobuf=3
diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml
@@ -9,6 +9,6 @@ LINUX_VER:
 - ubuntu18.04
 
 RAPIDS_VER:
-- "23.06"
+- "23.08"
 
 excludes:
diff --git a/continuous_integration/gpuci/environment-3.10.yaml b/continuous_integration/gpuci/environment-3.10.yaml
@@ -7,7 +7,7 @@ channels:
 - nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0
+- dask>=2022.3.0,<=2023.5.1
 # FIXME: handling is needed for httpx-based fastapi>=0.87.0
 - fastapi>=0.69.0,<0.87.0
 - fugue>=0.7.3
@@ -17,6 +17,7 @@ dependencies:
 - maturin>=0.12.8
 - mlflow
 - mock
+- numpy>=1.21.6
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
@@ -33,18 +34,15 @@ dependencies:
 - setuptools-rust>=1.5.2
 - sphinx
 - sqlalchemy<2
-- tpot
+- tpot>=0.12.0
 - tzlocal>=2.1
 - uvicorn>=0.13.4
 # GPU-specific requirements
 - cudatoolkit=11.5
-- cudf=23.06
-- cuml=23.06
-- dask-cudf=23.06
-- dask-cuda=23.06
-# tpot imports fail with numpy >=1.24.0
-# https://github.com/EpistasisLab/tpot/issues/1281
-- numpy>=1.20.1, <1.24.0
+- cudf=23.08
+- cuml=23.08
+- dask-cudf=23.08
+- dask-cuda=23.08
 - ucx-proc=*=gpu
-- ucx-py=0.32
-- xgboost=*rapidsai23.06
+- ucx-py=0.33
+- xgboost=*rapidsai23.08
diff --git a/continuous_integration/gpuci/environment-3.9.yaml b/continuous_integration/gpuci/environment-3.9.yaml
@@ -7,7 +7,7 @@ channels:
 - nodefaults
 dependencies:
 - c-compiler
-- dask>=2022.3.0
+- dask>=2022.3.0,<=2023.5.1
 # FIXME: handling is needed for httpx-based fastapi>=0.87.0
 - fastapi>=0.69.0,<0.87.0
 - fugue>=0.7.3
@@ -17,6 +17,7 @@ dependencies:
 - maturin>=0.12.8
 - mlflow
 - mock
+- numpy>=1.21.6
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
@@ -33,18 +34,15 @@ dependencies:
 - setuptools-rust>=1.5.2
 - sphinx
 - sqlalchemy<2
-- tpot
+- tpot>=0.12.0
 - tzlocal>=2.1
 - uvicorn>=0.13.4
 # GPU-specific requirements
 - cudatoolkit=11.5
-- cudf=23.06
-- cuml=23.06
-- dask-cudf=23.06
-- dask-cuda=23.06
-# tpot imports fail with numpy >=1.24.0
-# https://github.com/EpistasisLab/tpot/issues/1281
-- numpy>=1.20.1, <1.24.0
+- cudf=23.08
+- cuml=23.08
+- dask-cudf=23.08
+- dask-cuda=23.08
 - ucx-proc=*=gpu
-- ucx-py=0.32
-- xgboost=*rapidsai23.06
+- ucx-py=0.33
+- xgboost=*rapidsai23.08
diff --git a/continuous_integration/recipe/conda_build_config.yaml b/continuous_integration/recipe/conda_build_config.yaml
@@ -1,8 +1,8 @@
-python:
-    - 3.8
-    - 3.9
-    - 3.10
 c_compiler_version:
     - 11
 rust_compiler_version:
     - 1.69
+libprotobuf:
+    - 3
+setuptools_rust:
+    - 1.5.2
diff --git a/continuous_integration/recipe/meta.yaml b/continuous_integration/recipe/meta.yaml
@@ -22,17 +22,19 @@ build:
 
 requirements:
   build:
+    - python                                 # [build_platform != target_platform]
+    - cross-python_{{ target_platform }}     # [build_platform != target_platform]
+    - libprotobuf                            # [build_platform != target_platform]
     - {{ compiler('c') }}
     - {{ compiler('rust') }}
-    - setuptools-rust >=1.5.2
   host:
     - pip
     - python
-    - setuptools-rust >=1.5.2
-    - libprotobuf =3
+    - setuptools-rust
+    - libprotobuf
   run:
     - python
-    - dask >=2022.3.0
+    - dask >=2022.3.0,<=2023.5.1
     - pandas >=1.4.0
     # FIXME: handling is needed for httpx-based fastapi>=0.87.0
     - fastapi >=0.69.0,<0.87.0
@@ -41,6 +43,12 @@ requirements:
     - prompt-toolkit >=3.0.8
     - pygments >=2.7.1
     - tabulate
+  run_constrained:
+    # FIXME: aarch64 builds require glibc>=2.32 to prevent TLS allocation error when importing;
+    # is there any way we could modify the TLS model of its shared object to avoid this?
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1722181
+    # xref: https://github.com/dask-contrib/dask-sql/issues/1169
+    - __glibc >=2.32  # [aarch64]
 
 test:
   imports: