Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

friendlier error messages for missing chunk managers #9676

Open
wants to merge 21 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
1f71dac
raise an error message while guessing if there's no chunkmanager avai…
keewis Aug 7, 2024
66ed78c
don't skip the no chunkmanager test if dask is not installed
keewis Oct 24, 2024
1dba7b0
Merge branch 'main' into no-chunkmanager
keewis Oct 24, 2024
04d605c
whats-new
keewis Oct 24, 2024
b1d4017
ensure at least one chunk manager is available
keewis Oct 24, 2024
ab335a9
Merge branch 'main' into no-chunkmanager
keewis Nov 7, 2024
4119473
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 7, 2024
f9d1fcc
remove additional blank line from a bad merge
keewis Nov 7, 2024
e3cd03e
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 7, 2024
c646c29
improve the wording
keewis Nov 8, 2024
7bb3433
switch to ImportError
dcherian Nov 18, 2024
5f27715
Merge branch 'main' into no-chunkmanager
TomNicholas Nov 18, 2024
8d6657e
Merge branch 'main' into no-chunkmanager
TomNicholas Nov 22, 2024
34309f6
raise a helpful `ImportError` for known chunk managers
keewis Nov 22, 2024
f923622
make sure the new `ImportError` is actually raised
keewis Nov 22, 2024
60adcde
check that the more specific error message is preferred
keewis Nov 22, 2024
a69e794
prefer the more specific error
keewis Nov 22, 2024
7dd86c8
Merge branch 'main' into no-chunkmanager
keewis Nov 22, 2024
eabd209
also use `ImportError` as indicator for `chunks=None`
keewis Nov 22, 2024
060e77a
Merge branch 'main' into no-chunkmanager
keewis Nov 25, 2024
6c72381
move and improve the whats-new entry
keewis Nov 25, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions doc/whats-new.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ v.2024.11.1 (unreleased)

New Features
~~~~~~~~~~~~
- Improve the error message raised when using chunked-array methods if no chunk manager is available or if the requested chunk manager is missing (:pull:`9676`)
By `Justus Magin <https://github.com/keewis>`_.


Breaking changes
Expand Down
2 changes: 1 addition & 1 deletion xarray/backends/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -1415,7 +1415,7 @@ def open_zarr(
) # attempt to import that parallel backend

chunks = {}
except ValueError:
except (ValueError, ImportError):
chunks = None

if kwargs:
Expand Down
17 changes: 16 additions & 1 deletion xarray/namedarray/parallelcompat.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ def compute(

T_ChunkedArray = TypeVar("T_ChunkedArray", bound=ChunkedArrayMixinProtocol)

known_chunkmanagers = {
"dask": "dask",
"cubed": "cubed-xarray",
"arkouda": "arkouda-xarray",
}


@functools.lru_cache(maxsize=1)
def list_chunkmanagers() -> dict[str, ChunkManagerEntrypoint[Any]]:
Expand Down Expand Up @@ -106,7 +112,16 @@ def guess_chunkmanager(
manager = OPTIONS["chunk_manager"]

if isinstance(manager, str):
if manager not in chunkmanagers:
if manager not in chunkmanagers and manager in known_chunkmanagers:
raise ImportError(
f"chunk manager {manager!r} is not available."
f" Please make sure {known_chunkmanagers[manager]} is installed and importable."
)
elif len(chunkmanagers) == 0:
raise ImportError(
"no chunk managers available. Try installing `dask` or another package that provides a chunk manager."
)
elif manager not in chunkmanagers:
raise ValueError(
f"unrecognized chunk manager {manager} - must be one of: {list(chunkmanagers)}"
)
Expand Down
29 changes: 24 additions & 5 deletions xarray/tests/test_parallelcompat.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@
ChunkManagerEntrypoint,
get_chunked_array_type,
guess_chunkmanager,
known_chunkmanagers,
list_chunkmanagers,
load_chunkmanagers,
)
from xarray.tests import has_dask, requires_dask
from xarray.tests import requires_dask


class DummyChunkedArray(np.ndarray):
Expand Down Expand Up @@ -158,7 +159,18 @@ def test_get_chunkmanger_via_set_options(self, register_dummy_chunkmanager) -> N
chunkmanager = guess_chunkmanager(None)
assert isinstance(chunkmanager, DummyChunkManager)

def test_fail_on_nonexistent_chunkmanager(self) -> None:
def test_fail_on_known_but_missing_chunkmanager(
self, register_dummy_chunkmanager, monkeypatch
) -> None:
monkeypatch.setitem(known_chunkmanagers, "test", "test-package")
with pytest.raises(
ImportError, match="chunk manager 'test' is not available.+test-package"
):
guess_chunkmanager("test")

def test_fail_on_nonexistent_chunkmanager(
self, register_dummy_chunkmanager
) -> None:
with pytest.raises(ValueError, match="unrecognized chunk manager foo"):
dcherian marked this conversation as resolved.
Show resolved Hide resolved
guess_chunkmanager("foo")

Expand All @@ -167,9 +179,16 @@ def test_get_dask_if_installed(self) -> None:
chunkmanager = guess_chunkmanager(None)
assert isinstance(chunkmanager, DaskManager)

@pytest.mark.skipif(has_dask, reason="requires dask not to be installed")
def test_dont_get_dask_if_not_installed(self) -> None:
with pytest.raises(ValueError, match="unrecognized chunk manager dask"):
def test_no_chunk_manager_available(self, monkeypatch) -> None:
monkeypatch.setattr("xarray.namedarray.parallelcompat.list_chunkmanagers", dict)
with pytest.raises(ImportError, match="no chunk managers available"):
guess_chunkmanager("foo")

def test_no_chunk_manager_available_but_known_manager_requested(
self, monkeypatch
) -> None:
monkeypatch.setattr("xarray.namedarray.parallelcompat.list_chunkmanagers", dict)
with pytest.raises(ImportError, match="chunk manager 'dask' is not available"):
guess_chunkmanager("dask")

@requires_dask
Expand Down
Loading