From 10f1deefaab371ab48e759ced5fdeeb78a36b0dc Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 12 Jan 2024 12:06:21 -0500 Subject: [PATCH 1/3] Remove usages of rapids-env-update (#1304) Reference: https://github.com/rapidsai/ops/issues/2766 Replace rapids-env-update with rapids-configure-conda-channels, rapids-configure-sccache, and rapids-date-string. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/dask-cuda/pull/1304 --- ci/build_python.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 23c806704..1883ccf72 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -3,7 +3,11 @@ set -euo pipefail -source rapids-env-update +rapids-configure-conda-channels + +source rapids-configure-sccache + +source rapids-date-string export CMAKE_GENERATOR=Ninja From 34e7404731bc4517bd77dfb93eab04fddc28e29d Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Thu, 18 Jan 2024 16:02:16 +0000 Subject: [PATCH 2/3] Fix get_device_memory_ids (#1305) A recent change to the way `StringColumn`s are implemented in cudf threw up that we were never correctly determining the number of device buffers belonging to cudf columns if they had children (e.g. list and struct columns) or masks (any nullable column). Handle those cases and update the test. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - Peter Andreas Entschev (https://github.com/pentschev) - Mads R. B. Kristensen (https://github.com/madsbk) URL: https://github.com/rapidsai/dask-cuda/pull/1305 --- dask_cuda/get_device_memory_objects.py | 4 ++++ dask_cuda/tests/test_proxify_host_file.py | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dask_cuda/get_device_memory_objects.py b/dask_cuda/get_device_memory_objects.py index c5746c862..cd079f4ed 100644 --- a/dask_cuda/get_device_memory_objects.py +++ b/dask_cuda/get_device_memory_objects.py @@ -124,6 +124,10 @@ def get_device_memory_objects_cudf_index(obj): def get_device_memory_objects_cudf_multiindex(obj): return dispatch(obj._columns) + @dispatch.register(cudf.core.column.ColumnBase) + def get_device_memory_objects_cudf_column(obj): + return dispatch(obj.data) + dispatch(obj.children) + dispatch(obj.mask) + @sizeof.register_lazy("cupy") def register_cupy(): # NB: this overwrites dask.sizeof.register_cupy() diff --git a/dask_cuda/tests/test_proxify_host_file.py b/dask_cuda/tests/test_proxify_host_file.py index b1c9a9d52..2683ea36d 100644 --- a/dask_cuda/tests/test_proxify_host_file.py +++ b/dask_cuda/tests/test_proxify_host_file.py @@ -302,13 +302,24 @@ def test_dataframes_share_dev_mem(root_dir): def test_cudf_get_device_memory_objects(): cudf = pytest.importorskip("cudf") objects = [ - cudf.DataFrame({"a": range(10), "b": range(10)}, index=reversed(range(10))), + cudf.DataFrame( + {"a": [0, 1, 2, 3, None, 5, 6, 7, 8, 9], "b": range(10)}, + index=reversed(range(10)), + ), cudf.MultiIndex( levels=[[1, 2], ["blue", "red"]], codes=[[0, 0, 1, 1], [1, 0, 1, 0]] ), ] res = get_device_memory_ids(objects) - assert len(res) == 4, "We expect four buffer objects" + # Buffers are: + # 1. int data for objects[0].a + # 2. mask data for objects[0].a + # 3. int data for objects[0].b + # 4. int data for objects[0].index + # 5. int data for objects[1].levels[0] + # 6. char data for objects[1].levels[1] + # 7. offset data for objects[1].levels[1] + assert len(res) == 7, "We expect seven buffer objects" def test_externals(root_dir): From 55a3f1219f02416c2752983fd4596fb91f5fed4b Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Thu, 18 Jan 2024 14:44:09 -0500 Subject: [PATCH 3/3] DOC v24.04 Updates [skip ci] --- .github/workflows/build.yaml | 10 +++++----- .github/workflows/pr.yaml | 12 ++++++------ .github/workflows/test.yaml | 2 +- VERSION | 2 +- ci/build_docs.sh | 2 +- conda/environments/all_cuda-114_arch-x86_64.yaml | 14 +++++++------- conda/environments/all_cuda-118_arch-x86_64.yaml | 14 +++++++------- conda/environments/all_cuda-120_arch-x86_64.yaml | 14 +++++++------- dependencies.yaml | 14 +++++++------- pyproject.toml | 10 +++++----- 10 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8679cd61b..5d7cf1991 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: conda-python-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: if: github.ref_type == 'branch' needs: [conda-python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: upload-conda: needs: [conda-python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -59,7 +59,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -71,7 +71,7 @@ jobs: wheel-publish: needs: wheel-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8d701e0ae..fee5518b0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,26 +18,26 @@ jobs: - docs-build - wheel-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.04 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.04 conda-python-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -46,7 +46,7 @@ jobs: run_script: "ci/build_docs.sh" wheel-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 with: build_type: pull-request # Package is pure Python and only ever requires one build. diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9bf068690..011517e1a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/VERSION b/VERSION index 3c6c5e2b7..4a2fe8aa5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.02.00 +24.04.00 diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 2abfce6bd..1a66f3da6 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -23,7 +23,7 @@ rapids-mamba-retry install \ --channel "${PYTHON_CHANNEL}" \ dask-cuda -export RAPIDS_VERSION_NUMBER="24.02" +export RAPIDS_VERSION_NUMBER="24.04" export RAPIDS_DOCS_DIR="$(mktemp -d)" rapids-logger "Build Python docs" diff --git a/conda/environments/all_cuda-114_arch-x86_64.yaml b/conda/environments/all_cuda-114_arch-x86_64.yaml index 4b76616ab..ab0c276ee 100644 --- a/conda/environments/all_cuda-114_arch-x86_64.yaml +++ b/conda/environments/all_cuda-114_arch-x86_64.yaml @@ -10,10 +10,10 @@ dependencies: - click >=8.1 - cuda-version=11.4 - cudatoolkit -- cudf==24.2.* -- dask-cudf==24.2.* -- distributed-ucxx==0.36.* -- kvikio==24.2.* +- cudf==24.4.* +- dask-cudf==24.4.* +- distributed-ucxx==0.37.* +- kvikio==24.4.* - numactl-devel-cos7-x86_64 - numba>=0.57 - numpy>=1.21 @@ -24,13 +24,13 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.11 -- rapids-dask-dependency==24.2.* +- rapids-dask-dependency==24.4.* - setuptools>=64.0.0 - sphinx - sphinx-click>=2.7.1 - sphinx-rtd-theme>=0.5.1 - ucx-proc=*=gpu -- ucx-py==0.36.* -- ucxx==0.36.* +- ucx-py==0.37.* +- ucxx==0.37.* - zict>=2.0.0 name: all_cuda-114_arch-x86_64 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index bb23025eb..93ca69991 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -10,10 +10,10 @@ dependencies: - click >=8.1 - cuda-version=11.8 - cudatoolkit -- cudf==24.2.* -- dask-cudf==24.2.* -- distributed-ucxx==0.36.* -- kvikio==24.2.* +- cudf==24.4.* +- dask-cudf==24.4.* +- distributed-ucxx==0.37.* +- kvikio==24.4.* - numactl-devel-cos7-x86_64 - numba>=0.57 - numpy>=1.21 @@ -24,13 +24,13 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.11 -- rapids-dask-dependency==24.2.* +- rapids-dask-dependency==24.4.* - setuptools>=64.0.0 - sphinx - sphinx-click>=2.7.1 - sphinx-rtd-theme>=0.5.1 - ucx-proc=*=gpu -- ucx-py==0.36.* -- ucxx==0.36.* +- ucx-py==0.37.* +- ucxx==0.37.* - zict>=2.0.0 name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index a0dec45da..bddc705c0 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -11,10 +11,10 @@ dependencies: - cuda-nvcc-impl - cuda-nvrtc - cuda-version=12.0 -- cudf==24.2.* -- dask-cudf==24.2.* -- distributed-ucxx==0.36.* -- kvikio==24.2.* +- cudf==24.4.* +- dask-cudf==24.4.* +- distributed-ucxx==0.37.* +- kvikio==24.4.* - numactl-devel-cos7-x86_64 - numba>=0.57 - numpy>=1.21 @@ -25,13 +25,13 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.11 -- rapids-dask-dependency==24.2.* +- rapids-dask-dependency==24.4.* - setuptools>=64.0.0 - sphinx - sphinx-click>=2.7.1 - sphinx-rtd-theme>=0.5.1 - ucx-proc=*=gpu -- ucx-py==0.36.* -- ucxx==0.36.* +- ucx-py==0.37.* +- ucxx==0.37.* - zict>=2.0.0 name: all_cuda-120_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index eb7148615..671f43c48 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -148,23 +148,23 @@ dependencies: - numpy>=1.21 - pandas>=1.3,<1.6.0.dev0 - pynvml>=11.0.0,<11.5 - - rapids-dask-dependency==24.2.* + - rapids-dask-dependency==24.4.* - zict>=2.0.0 test_python: common: - output_types: [conda, requirements, pyproject] packages: - - cudf==24.2.* - - dask-cudf==24.2.* - - kvikio==24.2.* + - cudf==24.4.* + - dask-cudf==24.4.* + - kvikio==24.4.* - pytest - pytest-cov - - ucx-py==0.36.* + - ucx-py==0.37.* - output_types: [conda] packages: - - distributed-ucxx==0.36.* + - distributed-ucxx==0.37.* - ucx-proc=*=gpu - - ucxx==0.36.* + - ucxx==0.37.* specific: - output_types: conda matrices: diff --git a/pyproject.toml b/pyproject.toml index 6668e3f93..a6df9b9b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "numpy>=1.21", "pandas>=1.3,<1.6.0.dev0", "pynvml>=11.0.0,<11.5", - "rapids-dask-dependency==24.2.*", + "rapids-dask-dependency==24.4.*", "zict>=2.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -49,12 +49,12 @@ docs = [ "sphinx-rtd-theme>=0.5.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. test = [ - "cudf==24.2.*", - "dask-cudf==24.2.*", - "kvikio==24.2.*", + "cudf==24.4.*", + "dask-cudf==24.4.*", + "kvikio==24.4.*", "pytest", "pytest-cov", - "ucx-py==0.36.*", + "ucx-py==0.37.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls]