From e5b240c5f30e414388b5ca0e5a5a5c8e594d0ade Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Mon, 6 Nov 2023 16:34:41 +0100 Subject: [PATCH 1/7] Remove `ucp.reset()` requirement from `test_dgx` (#1269) By moving the `ucp.get_transports()` call to the subprocess we remove the requirement to reset UCX from the `pytest` process, preventing potential interferences with tests that run after. Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/dask-cuda/pull/1269 --- dask_cuda/tests/test_dgx.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/dask_cuda/tests/test_dgx.py b/dask_cuda/tests/test_dgx.py index 1fd6d0ebb..a7b79f327 100644 --- a/dask_cuda/tests/test_dgx.py +++ b/dask_cuda/tests/test_dgx.py @@ -128,7 +128,7 @@ def test_tcp_only(): def _test_ucx_infiniband_nvlink( - protocol, enable_infiniband, enable_nvlink, enable_rdmacm + skip_queue, protocol, enable_infiniband, enable_nvlink, enable_rdmacm ): cupy = pytest.importorskip("cupy") if protocol == "ucx": @@ -136,6 +136,14 @@ def _test_ucx_infiniband_nvlink( elif protocol == "ucxx": ucp = pytest.importorskip("ucxx") + if enable_infiniband and not any( + [at.startswith("rc") for at in ucp.get_active_transports()] + ): + skip_queue.put("No support available for 'rc' transport in UCX") + return + else: + skip_queue.put("ok") + if enable_infiniband is None and enable_nvlink is None and enable_rdmacm is None: enable_tcp_over_ucx = None cm_tls = ["all"] @@ -205,17 +213,16 @@ def check_ucx_options(): ) def test_ucx_infiniband_nvlink(protocol, params): if protocol == "ucx": - ucp = pytest.importorskip("ucp") + pytest.importorskip("ucp") elif protocol == "ucxx": - ucp = pytest.importorskip("ucxx") + pytest.importorskip("ucxx") - if params["enable_infiniband"]: - if not any([at.startswith("rc") for at in ucp.get_active_transports()]): - pytest.skip("No support available for 'rc' transport in UCX") + skip_queue = mp.Queue() p = mp.Process( target=_test_ucx_infiniband_nvlink, args=( + skip_queue, protocol, params["enable_infiniband"], params["enable_nvlink"], @@ -225,9 +232,8 @@ def test_ucx_infiniband_nvlink(protocol, params): p.start() p.join() - # Starting a new cluster on the same pytest process after an rdmacm cluster - # has been used may cause UCX-Py to complain about being already initialized. - if params["enable_rdmacm"] is True: - ucp.reset() + skip_msg = skip_queue.get() + if skip_msg != "ok": + pytest.skip(skip_msg) assert not p.exitcode From 9a5d06d8ec2318a0f23931188775802c227fd9b2 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 13 Nov 2023 10:24:53 -0600 Subject: [PATCH 2/7] Use new `rapids-dask-dependency` metapackage for managing dask versions (#1270) Currently dask versions are pinned as part of every release cycle and then unpinned for the next development cycle across all of RAPIDS. This introduces a great deal of churn. To centralize the dependency, we have created a metapackage to manage the required dask version and this PR introduces that metapackage as a dependency of dask-cuda. xref: https://github.com/rapidsai/cudf/pull/14364 Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Peter Andreas Entschev (https://github.com/pentschev) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/dask-cuda/pull/1270 --- ci/release/update-version.sh | 1 + conda/recipes/dask-cuda/meta.yaml | 1 - dependencies.yaml | 6 +----- pyproject.toml | 3 +-- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 94cd5d12b..e57e8b1e4 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -39,6 +39,7 @@ sed_runner "s/cudf=.*/cudf=${NEXT_SHORT_TAG}/g" dependencies.yaml sed_runner "s/dask-cudf=.*/dask-cudf=${NEXT_SHORT_TAG}/g" dependencies.yaml sed_runner "s/kvikio=.*/kvikio=${NEXT_SHORT_TAG}/g" dependencies.yaml sed_runner "s/ucx-py=.*/ucx-py=${NEXT_UCXPY_VERSION}/g" dependencies.yaml +sed_runner "s/rapids-dask-dependency=.*/rapids-dask-dependency=${NEXT_SHORT_TAG}.*/g" dependencies.yaml # CI files for FILE in .github/workflows/*.yaml; do diff --git a/conda/recipes/dask-cuda/meta.yaml b/conda/recipes/dask-cuda/meta.yaml index 6804b1ce4..c194d117b 100644 --- a/conda/recipes/dask-cuda/meta.yaml +++ b/conda/recipes/dask-cuda/meta.yaml @@ -32,7 +32,6 @@ requirements: - tomli run: - python - - dask-core >=2023.9.2 {% for r in data.get("project", {}).get("dependencies", []) %} - {{ r }} {% endfor %} diff --git a/dependencies.yaml b/dependencies.yaml index 02783dbff..b1c9cd3fc 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -101,16 +101,12 @@ dependencies: common: - output_types: [conda, requirements] packages: - - dask>=2023.9.2 - - distributed>=2023.9.2 - numba>=0.57 - numpy>=1.21 - pandas>=1.3,<1.6.0dev0 - pynvml>=11.0.0,<11.5 + - rapids-dask-dependency==23.12.* - zict>=2.0.0 - - output_types: [conda] - packages: - - dask-core>=2023.9.2 test_python: common: - output_types: [conda] diff --git a/pyproject.toml b/pyproject.toml index c240e61b7..3b02debbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,12 +16,11 @@ authors = [ license = { text = "Apache-2.0" } requires-python = ">=3.9" dependencies = [ - "dask >=2023.9.2", - "distributed >=2023.9.2", "pynvml >=11.0.0,<11.5", "numpy >=1.21", "numba >=0.57", "pandas >=1.3,<1.6.0dev0", + "rapids-dask-dependency==23.12.*", "zict >=2.0.0", ] classifiers = [ From d026d6eb497f9748f943f34118483a2dd4f2d32a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 13 Nov 2023 19:20:28 -0800 Subject: [PATCH 3/7] Add missing alpha spec (#1273) Without this extra spec, consumers of dask-cuda nightlies won't know that dask-cuda nightlies want to use nightlies of rapids-dask-dependency. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/dask-cuda/pull/1273 --- ci/build_python_pypi.sh | 10 ++++++++++ ci/test_python.sh | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ci/build_python_pypi.sh b/ci/build_python_pypi.sh index b13783d16..66bc7cf2e 100755 --- a/ci/build_python_pypi.sh +++ b/ci/build_python_pypi.sh @@ -17,6 +17,16 @@ if ! rapids-is-release-build; then export PACKAGE_VERSION_NUMBER="${version}" fi +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" pyproject.toml echo "${version}" | tr -d '"' > VERSION sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_name}/_version.py" diff --git a/ci/test_python.sh b/ci/test_python.sh index ca4140bae..f700c935b 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -55,7 +55,7 @@ timeout 60m pytest \ --cov=dask_cuda \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/dask-cuda-coverage.xml" \ --cov-report=term \ - tests + tests -k "not ucxx" popd rapids-logger "Run local benchmark" From 21e11bfa6f1c1231856c8a24ed9733a82ec55168 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 15 Nov 2023 18:45:50 -0800 Subject: [PATCH 4/7] Fix path (#1277) The `package_name` variable was not set here, so the git commit was never actually overwritten. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/dask-cuda/pull/1277 --- ci/build_python_pypi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_python_pypi.sh b/ci/build_python_pypi.sh index 66bc7cf2e..e177acf87 100755 --- a/ci/build_python_pypi.sh +++ b/ci/build_python_pypi.sh @@ -29,7 +29,7 @@ fi sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" pyproject.toml echo "${version}" | tr -d '"' > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_name}/_version.py" +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "dask_cuda/_version.py" # Compute/export RAPIDS_DATE_STRING source rapids-env-update From dc759b0a93caca35f44751b5713db88f36c66f66 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 17 Nov 2023 10:37:00 -0600 Subject: [PATCH 5/7] Enable build concurrency for nightly and merge triggers. (#1282) --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6e5f77d9b..f365c52c7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,7 +22,7 @@ on: default: nightly concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: From b1f13fc11a9c47e19ae950bfc0051e3df694a3ad Mon Sep 17 00:00:00 2001 From: Jacob Tomlinson Date: Mon, 20 Nov 2023 17:53:09 +0000 Subject: [PATCH 6/7] Set minimum click to 8.1 (#1272) Closes #1271 Authors: - Jacob Tomlinson (https://github.com/jacobtomlinson) - Lawrence Mitchell (https://github.com/wence-) Approvers: - Benjamin Zaitlen (https://github.com/quasiben) URL: https://github.com/rapidsai/dask-cuda/pull/1272 --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3b02debbe..c5286410a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,13 +13,14 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache-2.0" } +license = { text = "Apache-2.0." } requires-python = ">=3.9" dependencies = [ + "click >=8.1", "pynvml >=11.0.0,<11.5", "numpy >=1.21", "numba >=0.57", - "pandas >=1.3,<1.6.0dev0", + "pandas >=1.3,<1.6.0.dev0", "rapids-dask-dependency==23.12.*", "zict >=2.0.0", ] From be3170d17d86a057b322205434260b8451f88d56 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 20 Nov 2023 14:38:51 -0800 Subject: [PATCH 7/7] Fix license [skip ci] (#1285) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c5286410a..4f0da0689 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache-2.0." } +license = { text = "Apache-2.0" } requires-python = ">=3.9" dependencies = [ "click >=8.1",