Merge remote-tracking branch 'upstream/main' into pr/charlesbluca/1143

.github/CODEOWNERS

@@ -2,4 +2,7 @@
 *              @ayushdg @charlesbluca @galipremsagar
 
 # rust codeowners
-dask_planner/  @ayushdg @charlesbluca @galipremsagar @jdye64
+.cargo/        @ayushdg @charlesbluca @galipremsagar @jdye64
+src/           @ayushdg @charlesbluca @galipremsagar @jdye64
+Cargo.toml     @ayushdg @charlesbluca @galipremsagar @jdye64
+Cargo.lock     @ayushdg @charlesbluca @galipremsagar @jdye64

.github/dependabot.yml

@@ -1,7 +1,7 @@
 version: 2
 updates:
   - package-ecosystem: "cargo"
-    directory: "/dask_planner"
+    directory: "/"
     schedule:
       interval: "daily"
     ignore:
@@ -18,6 +18,5 @@ updates:
       # Check for updates to GitHub Actions every weekday
       interval: "weekly"
     ignore:
-      # ignore cibw patch updates
+      # prefer updating cibuildwheel manually as needed
       - dependency-name: "pypa/cibuildwheel"
-        update-types: ["version-update:semver-patch"]

.github/workflows/conda.yml

@@ -3,14 +3,11 @@ on:
   push:
     branches:
       - main
-      - datafusion-sql-planner
   pull_request:
     paths:
-      - setup.py
-      - dask_planner/Cargo.toml
-      - dask_planner/Cargo.lock
-      - dask_planner/pyproject.toml
-      - dask_planner/rust-toolchain.toml
+      - Cargo.toml
+      - Cargo.lock
+      - pyproject.toml
       - continuous_integration/recipe/**
       - .github/workflows/conda.yml
   schedule:
@@ -29,9 +26,44 @@ defaults:
 
 jobs:
   conda:
-    name: Build (and optionally upload) the conda nightly
+    name: "Build conda nightlies (python: ${{ matrix.python }}, arch: ${{ matrix.arch }})"
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.8", "3.9", "3.10"]
+        arch: ["linux-64", "linux-aarch64"]
     steps:
+      - name: Manage disk space
+        if: matrix.arch == 'linux-aarch64'
+        run: |
+          sudo mkdir -p /opt/empty_dir || true
+          for d in \
+                    /opt/ghc \
+                    /opt/hostedtoolcache \
+                    /usr/lib/jvm \
+                    /usr/local/.ghcup \
+                    /usr/local/lib/android \
+                    /usr/local/share/powershell \
+                    /usr/share/dotnet \
+                    /usr/share/swift \
+                    ; do
+            sudo rsync --stats -a --delete /opt/empty_dir/ $d || true
+          done
+          sudo apt-get purge -y -f firefox \
+                                    google-chrome-stable \
+                                    microsoft-edge-stable
+          sudo apt-get autoremove -y >& /dev/null
+          sudo apt-get autoclean -y >& /dev/null
+          sudo docker image prune --all --force
+          df -h
+      - name: Create swapfile
+        if: matrix.arch == 'linux-aarch64'
+        run: |
+          sudo fallocate -l 10GiB /swapfile || true
+          sudo chmod 600 /swapfile || true
+          sudo mkswap /swapfile || true
+          sudo swapon /swapfile || true
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -49,23 +81,36 @@ jobs:
           which python
           pip list
           mamba list
-      - name: Build conda package
+      - name: Build conda packages
         run: |
           # suffix for nightly package versions
           export VERSION_SUFFIX=a`date +%y%m%d`
 
           conda mambabuild continuous_integration/recipe \
+                           --python ${{ matrix.python }} \
+                           --variants "{target_platform: [${{ matrix.arch }}]}" \
+                           --error-overlinking \
+                           --no-test \
                            --no-anaconda-upload \
-                           --output-folder .
-      - name: Upload conda package
+                           --output-folder packages
+      - name: Test conda packages
+        if: matrix.arch == 'linux-64'  # can only test native platform packages
+        run: |
+          conda mambabuild --test packages/${{ matrix.arch }}/*.tar.bz2
+      - name: Upload conda packages as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: "conda nightlies (python - ${{ matrix.python }}, arch - ${{ matrix.arch }})"
+          # need to install all conda channel metadata to properly install locally
+          path: packages/
+      - name: Upload conda packages to Anaconda
         if: |
           github.event_name == 'push'
           && github.repository == 'dask-contrib/dask-sql'
         env:
           ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
-          LABEL: ${{ github.ref == 'refs/heads/datafusion-sql-planner' && 'dev_datafusion' || 'dev' }}
         run: |
           # install anaconda for upload
           mamba install -c conda-forge anaconda-client
 
-          anaconda upload --label $LABEL linux-64/*.tar.bz2
+          anaconda upload --label dev packages/${{ matrix.arch }}/*.tar.bz2

.github/workflows/docker.yml

@@ -1,11 +1,18 @@
-name: Upload Docker image
+name: Build Docker image
 
 on:
   release:
     types: [created]
   push:
     branches:
       - main
+  pull_request:
+    paths:
+      - Cargo.toml
+      - Cargo.lock
+      - pyproject.toml
+      - docker/**
+      - .github/workflows/docker.yml
 
 # When this workflow is queued, automatically cancel any previous running
 # or pending jobs from the same branch
@@ -17,15 +24,16 @@ jobs:
   push_to_registry:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
-    if: github.repository == 'dask-contrib/dask-sql'
+    env:
+      DOCKER_PUSH: ${{ contains(['push', 'release'], github.event_name) && github.repository == 'dask-contrib/dask-sql' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ["linux/amd64", "linux/arm64", "linux/386"]
     steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+      - uses: actions/checkout@v3
       - name: Login to DockerHub
+        if: ${{ fromJSON(env.DOCKER_PUSH) }}
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
@@ -41,10 +49,16 @@ jobs:
           context: .
           file: ./docker/main.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
-          platforms: linux/amd64,linux/arm64,linux/386
+          platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_main.outputs.tags }}
           labels: ${{ steps.docker_meta_main.outputs.labels }}
-          push: true
+          push: ${{ fromJSON(env.DOCKER_PUSH) }}
+          load: ${{ !fromJSON(env.DOCKER_PUSH) }}
+      - name: Check images
+        run: |
+          df -h
+          docker image ls
+          docker image inspect ${{ steps.docker_meta_main.outputs.tags }}
       - name: Docker meta for cloud image
         id: docker_meta_cloud
         uses: crazy-max/ghaction-docker-meta@v4
@@ -56,7 +70,8 @@ jobs:
           context: .
           file: ./docker/cloud.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
-          platforms: linux/amd64,linux/arm64,linux/386
+          platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_cloud.outputs.tags }}
           labels: ${{ steps.docker_meta_cloud.outputs.labels }}
-          push: true
+          push: ${{ fromJSON(env.DOCKER_PUSH) }}
+          load: ${{ !fromJSON(env.DOCKER_PUSH) }}