From 1f71daced8f96e4cafccacb802dde1ccb7cdb48a Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Wed, 7 Aug 2024 15:45:03 +0200 Subject: [PATCH 01/15] raise an error message while guessing if there's no chunkmanager available --- xarray/namedarray/parallelcompat.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index dd555fe200a..ac2fa6465dd 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -103,6 +103,10 @@ def guess_chunkmanager( """ chunkmanagers = list_chunkmanagers() + if len(chunkmanagers) == 0: + raise ValueError( + "no chunk managers available. Try installing `dask` or a package that provides a chunk manager." + ) if manager is None: if len(chunkmanagers) == 1: From 66ed78cb5af2b9b73f474077b85f369691a14635 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 24 Oct 2024 23:12:14 +0200 Subject: [PATCH 02/15] don't skip the no chunkmanager test if dask is not installed --- xarray/tests/test_parallelcompat.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index dbe40be710c..2c7b9374eb9 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -16,7 +16,7 @@ list_chunkmanagers, load_chunkmanagers, ) -from xarray.tests import has_dask, requires_dask +from xarray.tests import requires_dask class DummyChunkedArray(np.ndarray): @@ -161,9 +161,11 @@ def test_get_dask_if_installed(self) -> None: chunkmanager = guess_chunkmanager(None) assert isinstance(chunkmanager, DaskManager) - @pytest.mark.skipif(has_dask, reason="requires dask not to be installed") - def test_dont_get_dask_if_not_installed(self) -> None: - with pytest.raises(ValueError, match="unrecognized chunk manager dask"): + def test_no_chunk_manager_available(self, monkeypatch) -> None: + monkeypatch.setattr( + "xarray.namedarray.parallelcompat.list_chunkmanagers", lambda: {} + ) + with pytest.raises(ValueError, match="no chunk managers available"): guess_chunkmanager("dask") @requires_dask From 04d605cbbb68aa392b13afed06b4591e308d6408 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 24 Oct 2024 23:19:59 +0200 Subject: [PATCH 03/15] whats-new --- doc/whats-new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index c3dd6776c27..ec3ca36ef56 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -76,6 +76,8 @@ Bug fixes `_. - Fix binning by multiple variables where some bins have no observations. (:issue:`9630`). By `Deepak Cherian `_. +- Improve the error message raised when using chunked-array methods if no chunk manager is available (:pull:`9676`) + By `Justus Magin `_. Documentation ~~~~~~~~~~~~~ From b1d40179b075c0bac97375e7b03f0db27a8f27ad Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 24 Oct 2024 23:30:12 +0200 Subject: [PATCH 04/15] ensure at least one chunk manager is available --- xarray/tests/test_parallelcompat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index 517d9fc342c..5cf62e5a317 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -158,7 +158,9 @@ def test_get_chunkmanger_via_set_options(self, register_dummy_chunkmanager) -> N chunkmanager = guess_chunkmanager(None) assert isinstance(chunkmanager, DummyChunkManager) - def test_fail_on_nonexistent_chunkmanager(self) -> None: + def test_fail_on_nonexistent_chunkmanager( + self, register_dummy_chunkmanager + ) -> None: with pytest.raises(ValueError, match="unrecognized chunk manager foo"): guess_chunkmanager("foo") From 4119473d1dae1e7ec95ea54562f33075e0ceb615 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:27:45 +0000 Subject: [PATCH 05/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_parallelcompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index 5cf62e5a317..04a2dff0f46 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -171,7 +171,7 @@ def test_get_dask_if_installed(self) -> None: def test_no_chunk_manager_available(self, monkeypatch) -> None: monkeypatch.setattr( - "xarray.namedarray.parallelcompat.list_chunkmanagers", lambda: {} + "xarray.namedarray.parallelcompat.list_chunkmanagers", dict ) with pytest.raises(ValueError, match="no chunk managers available"): guess_chunkmanager("dask") From f9d1fcccd0647240ed76f5f544d1fee589b0ef5c Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 7 Nov 2024 18:39:52 +0100 Subject: [PATCH 06/15] remove additional blank line from a bad merge --- doc/whats-new.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 93ae2394cc1..86c2eae66cf 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -135,7 +135,6 @@ Bug fixes - Fix issue where polyfit wouldn't handle non-dimension coordinates. (:issue:`4375`, :pull:`9369`) By `Karl Krauth `_. - Documentation ~~~~~~~~~~~~~ From e3cd03e7e1792c387536fb5f35651492f99b302b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:40:06 +0000 Subject: [PATCH 07/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_parallelcompat.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index 04a2dff0f46..415086da272 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -170,9 +170,7 @@ def test_get_dask_if_installed(self) -> None: assert isinstance(chunkmanager, DaskManager) def test_no_chunk_manager_available(self, monkeypatch) -> None: - monkeypatch.setattr( - "xarray.namedarray.parallelcompat.list_chunkmanagers", dict - ) + monkeypatch.setattr("xarray.namedarray.parallelcompat.list_chunkmanagers", dict) with pytest.raises(ValueError, match="no chunk managers available"): guess_chunkmanager("dask") From c646c29c7c9fd89b8bc03efb586ba0661d89fb3b Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 8 Nov 2024 20:54:53 +0100 Subject: [PATCH 08/15] improve the wording Co-authored-by: Tom Nicholas --- xarray/namedarray/parallelcompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index 97149500d78..2f37c62d674 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -97,7 +97,7 @@ def guess_chunkmanager( chunkmanagers = list_chunkmanagers() if len(chunkmanagers) == 0: raise ValueError( - "no chunk managers available. Try installing `dask` or a package that provides a chunk manager." + "no chunk managers available. Try installing `dask` or another package that provides a chunk manager." ) if manager is None: From 7bb34334e20534735d345243964818a6899ee5c5 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 18 Nov 2024 11:21:22 -0700 Subject: [PATCH 09/15] switch to ImportError --- xarray/namedarray/parallelcompat.py | 2 +- xarray/tests/test_parallelcompat.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index 2f37c62d674..f088040e376 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -96,7 +96,7 @@ def guess_chunkmanager( chunkmanagers = list_chunkmanagers() if len(chunkmanagers) == 0: - raise ValueError( + raise ImportError( "no chunk managers available. Try installing `dask` or another package that provides a chunk manager." ) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index 415086da272..f3f4f0f19b0 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -161,7 +161,7 @@ def test_get_chunkmanger_via_set_options(self, register_dummy_chunkmanager) -> N def test_fail_on_nonexistent_chunkmanager( self, register_dummy_chunkmanager ) -> None: - with pytest.raises(ValueError, match="unrecognized chunk manager foo"): + with pytest.raises(ImportError, match="unrecognized chunk manager foo"): guess_chunkmanager("foo") @requires_dask @@ -171,7 +171,7 @@ def test_get_dask_if_installed(self) -> None: def test_no_chunk_manager_available(self, monkeypatch) -> None: monkeypatch.setattr("xarray.namedarray.parallelcompat.list_chunkmanagers", dict) - with pytest.raises(ValueError, match="no chunk managers available"): + with pytest.raises(ImportError, match="no chunk managers available"): guess_chunkmanager("dask") @requires_dask From 34309f626600250ee27f7d3f26bf9b472f18cae7 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Nov 2024 22:06:38 +0100 Subject: [PATCH 10/15] raise a helpful `ImportError` for known chunk managers --- xarray/namedarray/parallelcompat.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index 06219cdf9e3..af771c6891f 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -46,6 +46,12 @@ def compute( T_ChunkedArray = TypeVar("T_ChunkedArray", bound=ChunkedArrayMixinProtocol) +known_chunkmanagers = { + "dask": "dask", + "cubed": "cubed-xarray", + "arkouda": "arkouda-xarray", +} + @functools.lru_cache(maxsize=1) def list_chunkmanagers() -> dict[str, ChunkManagerEntrypoint[Any]]: @@ -110,7 +116,12 @@ def guess_chunkmanager( manager = OPTIONS["chunk_manager"] if isinstance(manager, str): - if manager not in chunkmanagers: + if manager not in chunkmanagers and manager in known_chunkmanagers: + raise ImportError( + f"chunk manager {manager!r} is not available." + f" Please make sure {known_chunkmanagers[manager]} is installed and importable." + ) + elif manager not in chunkmanagers: raise ValueError( f"unrecognized chunk manager {manager} - must be one of: {list(chunkmanagers)}" ) From f923622aad247bbf33595acf5a0a16861d592f09 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Nov 2024 22:16:08 +0100 Subject: [PATCH 11/15] make sure the new `ImportError` is actually raised --- xarray/tests/test_parallelcompat.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index f3f4f0f19b0..1548441f8d9 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -14,6 +14,7 @@ ChunkManagerEntrypoint, get_chunked_array_type, guess_chunkmanager, + known_chunkmanagers, list_chunkmanagers, load_chunkmanagers, ) @@ -158,10 +159,19 @@ def test_get_chunkmanger_via_set_options(self, register_dummy_chunkmanager) -> N chunkmanager = guess_chunkmanager(None) assert isinstance(chunkmanager, DummyChunkManager) + def test_fail_on_known_but_missing_chunkmanager( + self, register_dummy_chunkmanager, monkeypatch + ) -> None: + monkeypatch.setitem(known_chunkmanagers, "test", "test-package") + with pytest.raises( + ImportError, match="chunk manager 'test' is not available.+test-package" + ): + guess_chunkmanager("test") + def test_fail_on_nonexistent_chunkmanager( self, register_dummy_chunkmanager ) -> None: - with pytest.raises(ImportError, match="unrecognized chunk manager foo"): + with pytest.raises(ValueError, match="unrecognized chunk manager foo"): guess_chunkmanager("foo") @requires_dask From 60adcde4c0d94d3621316c60ebdc42082f9a044f Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Nov 2024 22:19:23 +0100 Subject: [PATCH 12/15] check that the more specific error message is preferred --- xarray/tests/test_parallelcompat.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/xarray/tests/test_parallelcompat.py b/xarray/tests/test_parallelcompat.py index 1548441f8d9..4a99f88f7d6 100644 --- a/xarray/tests/test_parallelcompat.py +++ b/xarray/tests/test_parallelcompat.py @@ -182,6 +182,13 @@ def test_get_dask_if_installed(self) -> None: def test_no_chunk_manager_available(self, monkeypatch) -> None: monkeypatch.setattr("xarray.namedarray.parallelcompat.list_chunkmanagers", dict) with pytest.raises(ImportError, match="no chunk managers available"): + guess_chunkmanager("foo") + + def test_no_chunk_manager_available_but_known_manager_requested( + self, monkeypatch + ) -> None: + monkeypatch.setattr("xarray.namedarray.parallelcompat.list_chunkmanagers", dict) + with pytest.raises(ImportError, match="chunk manager 'dask' is not available"): guess_chunkmanager("dask") @requires_dask From a69e794317362a6e9c8ce0f3740e95cf99054572 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Nov 2024 22:20:28 +0100 Subject: [PATCH 13/15] prefer the more specific error --- xarray/namedarray/parallelcompat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/namedarray/parallelcompat.py b/xarray/namedarray/parallelcompat.py index af771c6891f..b6f2a0f5698 100644 --- a/xarray/namedarray/parallelcompat.py +++ b/xarray/namedarray/parallelcompat.py @@ -102,10 +102,6 @@ def guess_chunkmanager( """ chunkmanagers = list_chunkmanagers() - if len(chunkmanagers) == 0: - raise ImportError( - "no chunk managers available. Try installing `dask` or another package that provides a chunk manager." - ) if manager is None: if len(chunkmanagers) == 1: @@ -121,6 +117,10 @@ def guess_chunkmanager( f"chunk manager {manager!r} is not available." f" Please make sure {known_chunkmanagers[manager]} is installed and importable." ) + elif len(chunkmanagers) == 0: + raise ImportError( + "no chunk managers available. Try installing `dask` or another package that provides a chunk manager." + ) elif manager not in chunkmanagers: raise ValueError( f"unrecognized chunk manager {manager} - must be one of: {list(chunkmanagers)}" From eabd2090e0c0b722a88e9f63e6e4eb97769fea2c Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Nov 2024 23:06:03 +0100 Subject: [PATCH 14/15] also use `ImportError` as indicator for `chunks=None` --- xarray/backends/zarr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 1acc0a502e6..244a82be7d3 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1415,7 +1415,7 @@ def open_zarr( ) # attempt to import that parallel backend chunks = {} - except ValueError: + except (ValueError, ImportError): chunks = None if kwargs: From 6c72381147a321ea4900118105610a184d7aca8b Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Mon, 25 Nov 2024 12:26:37 +0100 Subject: [PATCH 15/15] move and improve the whats-new entry --- doc/whats-new.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4ab66e93003..d9a0aaff687 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -21,6 +21,8 @@ v.2024.11.1 (unreleased) New Features ~~~~~~~~~~~~ +- Improve the error message raised when using chunked-array methods if no chunk manager is available or if the requested chunk manager is missing (:pull:`9676`) + By `Justus Magin `_. Breaking changes @@ -120,8 +122,6 @@ Bug fixes By `Stephan Hoyer `_. - Fix regression in the interoperability of :py:meth:`DataArray.polyfit` and :py:meth:`xr.polyval` for date-time coordinates. (:pull:`9691`). By `Pascal Bourgault `_. -- Improve the error message raised when using chunked-array methods if no chunk manager is available (:pull:`9676`) - By `Justus Magin `_. - Fix CF decoding of ``grid_mapping`` to allow all possible formats, add tests (:issue:`9761`, :pull:`9765`). By `Kai Mühlbauer `_. - Add ``User-Agent`` to request-headers when retrieving tutorial data (:issue:`9774`, :pull:`9782`)