From 0e93bec0c29738ff43a4babac27f73b15a2c008a Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Tue, 19 Mar 2024 10:28:29 -0400 Subject: [PATCH] rename classes --- .vscode/tasks/quick_test.sh | 4 +++ .vscode/tasks/test.sh | 4 +++ README.md | 4 +-- devel/old_tests/old_tests.py | 4 +-- devel/old_tests/test_lindi_client.py | 4 +-- devel/old_tests/test_with_real_data.py | 4 +-- devel/old_tests/tests.py | 18 +++++----- examples/example1.py | 18 +++++----- examples/try_pynwb.py | 4 +-- lindi/LindiH5Store/__init__.py | 1 - .../LindiH5ZarrStore.py} | 24 ++++++------- lindi/LindiH5ZarrStore/__init__.py | 1 + .../_h5_attr_to_zarr_attr.py | 2 +- .../_h5_filters_to_codecs.py | 0 .../_util.py | 0 .../_zarr_info_for_h5_dataset.py | 2 +- .../LindiAttributes.py | 0 .../LindiDataset.py | 0 .../LindiGroup.py | 0 .../LindiH5pyFile.py} | 22 ++++++------ .../LindiReference.py | 0 .../__init__.py | 2 +- lindi/__init__.py | 4 +-- tests/test_core.py | 36 +++++++++---------- tests/test_with_real_data.py | 8 ++--- 25 files changed, 87 insertions(+), 79 deletions(-) delete mode 100644 lindi/LindiH5Store/__init__.py rename lindi/{LindiH5Store/LindiH5Store.py => LindiH5ZarrStore/LindiH5ZarrStore.py} (97%) create mode 100644 lindi/LindiH5ZarrStore/__init__.py rename lindi/{LindiH5Store => LindiH5ZarrStore}/_h5_attr_to_zarr_attr.py (98%) rename lindi/{LindiH5Store => LindiH5ZarrStore}/_h5_filters_to_codecs.py (100%) rename lindi/{LindiH5Store => LindiH5ZarrStore}/_util.py (100%) rename lindi/{LindiH5Store => LindiH5ZarrStore}/_zarr_info_for_h5_dataset.py (99%) rename lindi/{LindiClient => LindiH5pyFile}/LindiAttributes.py (100%) rename lindi/{LindiClient => LindiH5pyFile}/LindiDataset.py (100%) rename lindi/{LindiClient => LindiH5pyFile}/LindiGroup.py (100%) rename lindi/{LindiClient/LindiClient.py => LindiH5pyFile/LindiH5pyFile.py} (86%) rename lindi/{LindiClient => LindiH5pyFile}/LindiReference.py (100%) rename lindi/{LindiClient => LindiH5pyFile}/__init__.py (79%) diff --git a/.vscode/tasks/quick_test.sh b/.vscode/tasks/quick_test.sh index 6d8a84a..173dd78 100755 --- a/.vscode/tasks/quick_test.sh +++ b/.vscode/tasks/quick_test.sh @@ -2,6 +2,10 @@ set -ex # black --check . + +cd lindi flake8 . # pyright +cd .. + pytest --cov=lindi --cov-report=xml --cov-report=term -m "not slow" tests/ diff --git a/.vscode/tasks/test.sh b/.vscode/tasks/test.sh index 0e09ca0..c87184e 100755 --- a/.vscode/tasks/test.sh +++ b/.vscode/tasks/test.sh @@ -2,6 +2,10 @@ set -ex # black --check . + +cd lindi flake8 . pyright +cd .. + pytest --cov=lindi --cov-report=xml --cov-report=term tests/ diff --git a/README.md b/README.md index 4fa55da..3272790 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ h5_url = "https://api.dandiarchive.org/api/assets/3d12a902-139a-4c1a-8fd0-0a7faf # Create the read-only Zarr store for the h5 file -store = lindi.LindiH5Store.from_file(h5_url) +store = lindi.LindiH5ZarrStore.from_file(h5_url) # Create the reference file system object rfs = store.to_reference_file_system() @@ -47,7 +47,7 @@ with open("example.zarr.json", "w") as f: json.dump(rfs, f, indent=2) # Create the h5py-like client from the reference file system -client = lindi.LindiClient.from_reference_file_system(rfs) +client = lindi.LindiH5pyFile.from_reference_file_system(rfs) # Try to read using pynwb # (This part does not work yet) diff --git a/devel/old_tests/old_tests.py b/devel/old_tests/old_tests.py index 6ea621f..de05052 100644 --- a/devel/old_tests/old_tests.py +++ b/devel/old_tests/old_tests.py @@ -4,7 +4,7 @@ import h5py import zarr import kerchunk.hdf # type: ignore -from lindi import LindiH5Store +from lindi import LindiH5ZarrStore from fsspec.implementations.reference import ReferenceFileSystem @@ -88,7 +88,7 @@ def test_numpy_array_of_strings(): def _get_lindi_zarr(filename): - store = LindiH5Store.from_file(filename, url='.') # use url='.' so that a reference file system can be created + store = LindiH5ZarrStore.from_file(filename, url='.') # use url='.' so that a reference file system can be created root = zarr.open(store) return root, store diff --git a/devel/old_tests/test_lindi_client.py b/devel/old_tests/test_lindi_client.py index 2fa62de..846217d 100644 --- a/devel/old_tests/test_lindi_client.py +++ b/devel/old_tests/test_lindi_client.py @@ -1,8 +1,8 @@ -from lindi import LindiClient, LindiGroup, LindiDataset +from lindi import LindiH5pyFile, LindiGroup, LindiDataset def test_lindi_client(): - client = LindiClient.from_file("example_0.zarr.json") + client = LindiH5pyFile.from_file("example_0.zarr.json") for k, v in client.attrs.items(): print(f"{k}: {v}") diff --git a/devel/old_tests/test_with_real_data.py b/devel/old_tests/test_with_real_data.py index 9de0291..9ef76a8 100644 --- a/devel/old_tests/test_with_real_data.py +++ b/devel/old_tests/test_with_real_data.py @@ -3,7 +3,7 @@ import zarr import h5py import remfile -from lindi import LindiH5Store +from lindi import LindiH5ZarrStore examples = [] @@ -40,7 +40,7 @@ def do_compare(example_num: int): h5_url = example["h5_url"] print(f"Running comparison for {h5_url}") h5f = h5py.File(remfile.File(h5_url), "r") - with LindiH5Store.from_file(h5_url) as store: + with LindiH5ZarrStore.from_file(h5_url) as store: root = zarr.open(store) assert isinstance(root, zarr.Group) diff --git a/devel/old_tests/tests.py b/devel/old_tests/tests.py index 8359136..9f77442 100644 --- a/devel/old_tests/tests.py +++ b/devel/old_tests/tests.py @@ -1,7 +1,7 @@ import tempfile import numpy as np import h5py -from lindi import LindiH5Store, LindiClient, LindiGroup, LindiDataset +from lindi import LindiH5ZarrStore, LindiH5pyFile, LindiGroup, LindiDataset def test_scalar_dataset(): @@ -11,11 +11,11 @@ def test_scalar_dataset(): filename = f"{tmpdir}/test.h5" with h5py.File(filename, "w") as f: f.create_dataset("X", data=val) - with LindiH5Store.from_file( + with LindiH5ZarrStore.from_file( filename, url=filename ) as store: # set url so that a reference file system can be created rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) h5f = h5py.File(filename, "r") X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) @@ -69,11 +69,11 @@ def test_numpy_array(): filename = f"{tmpdir}/test.h5" with h5py.File(filename, "w") as f: f.create_dataset("X", data=array, chunks=chunks) - with LindiH5Store.from_file( + with LindiH5ZarrStore.from_file( filename, url=filename ) as store: # set url so that a reference file system can be created rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) h5f = h5py.File(filename, "r") X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) @@ -92,9 +92,9 @@ def test_numpy_array_of_strings(): with h5py.File(filename, "w") as f: f.create_dataset("X", data=["abc", "def", "ghi"]) h5f = h5py.File(filename, "r") - with LindiH5Store.from_file(filename, url=filename) as store: + with LindiH5ZarrStore.from_file(filename, url=filename) as store: rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) X2 = client["X"] @@ -120,9 +120,9 @@ def test_attributes(): f["group"].attrs["foo"] = "bar2" f["group"].attrs["baz"] = 3.15 h5f = h5py.File(filename, "r") - with LindiH5Store.from_file(filename, url=filename) as store: + with LindiH5ZarrStore.from_file(filename, url=filename) as store: rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) X2 = client["X"] diff --git a/examples/example1.py b/examples/example1.py index b22de17..6721603 100644 --- a/examples/example1.py +++ b/examples/example1.py @@ -2,18 +2,18 @@ import h5py import tempfile import lindi -from lindi import LindiH5Store, LindiClient, LindiDataset +from lindi import LindiH5ZarrStore, LindiH5pyFile, LindiDataset from _check_equal import _check_equal def example1(): # In this example, we create a temporary hdf5 file, with some sample # datasets, groups, and attributes. We then load that file using - # LindiH5Store which is a zarr storage backend providing read-only view of + # LindiH5ZarrStore which is a zarr storage backend providing read-only view of # that hdf5 file. We then create a reference file system and use that to - # create a LindiClient, which mimics the h5py API. We then compare the + # create a LindiH5pyFile, which mimics the h5py API. We then compare the # datasets, groups, and attributes of the original hdf5 file with those of - # the LindiClient. + # the LindiH5pyFile. with tempfile.TemporaryDirectory() as tmpdir: print("Creating an example hdf5 file") @@ -31,16 +31,16 @@ def example1(): f["group"].attrs["baz"] = 3.15 h5f = h5py.File(filename, "r") - print("Creating a LindiH5Store from the hdf5 file") + print("Creating a LindiH5ZarrStore from the hdf5 file") # We set url to filename so that the references can point to a local file # but normally, this would be a remote URL - with LindiH5Store.from_file(filename, url=filename) as store: - print("Creating a reference file system from the LindiH5Store") + with LindiH5ZarrStore.from_file(filename, url=filename) as store: + print("Creating a reference file system from the LindiH5ZarrStore") rfs_fname = f"{tmpdir}/example.zarr.json" store.to_file(rfs_fname) - print("Creating a LindiClient from the reference file system") - client = LindiClient.from_file(rfs_fname) + print("Creating a LindiH5pyFile from the reference file system") + client = LindiH5pyFile.from_file(rfs_fname) print("Comparing dataset: X") X1 = h5f["X"] diff --git a/examples/try_pynwb.py b/examples/try_pynwb.py index 610b66c..32457ab 100644 --- a/examples/try_pynwb.py +++ b/examples/try_pynwb.py @@ -9,7 +9,7 @@ def try_pynwb(): # Create the read-only store for the h5 file - store = lindi.LindiH5Store.from_file(h5_url) + store = lindi.LindiH5ZarrStore.from_file(h5_url) # Create the reference file system object rfs = store.to_reference_file_system() @@ -19,7 +19,7 @@ def try_pynwb(): json.dump(rfs, f, indent=2) # Create the client from the reference file system - client = lindi.LindiClient.from_reference_file_system(rfs) + client = lindi.LindiH5pyFile.from_reference_file_system(rfs) # Try to read using pynwb with pynwb.NWBHDF5IO(file=client, mode="r") as io: diff --git a/lindi/LindiH5Store/__init__.py b/lindi/LindiH5Store/__init__.py deleted file mode 100644 index 858de62..0000000 --- a/lindi/LindiH5Store/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .LindiH5Store import LindiH5Store, LindiH5StoreOpts # noqa: F401 diff --git a/lindi/LindiH5Store/LindiH5Store.py b/lindi/LindiH5ZarrStore/LindiH5ZarrStore.py similarity index 97% rename from lindi/LindiH5Store/LindiH5Store.py rename to lindi/LindiH5ZarrStore/LindiH5ZarrStore.py index 114bb69..71ddd8c 100644 --- a/lindi/LindiH5Store/LindiH5Store.py +++ b/lindi/LindiH5ZarrStore/LindiH5ZarrStore.py @@ -18,22 +18,22 @@ @dataclass -class LindiH5StoreOpts: +class LindiH5ZarrStoreOpts: num_dataset_chunks_threshold: Union[int, None] = 1000 -class LindiH5Store(Store): +class LindiH5ZarrStore(Store): """A zarr store that provides a read-only view of an HDF5 file. Do not call the constructor directly. Instead do one of the following: - store = LindiH5Store.from_file(hdf5_file_name) + store = LindiH5ZarrStore.from_file(hdf5_file_name) # do stuff with store store.close() or - with LindiH5Store.from_file(hdf5_file_name) as store: + with LindiH5ZarrStore.from_file(hdf5_file_name) as store: # do stuff with store """ @@ -41,7 +41,7 @@ def __init__( self, *, _file: Union[IO, Any], - _opts: LindiH5StoreOpts, + _opts: LindiH5ZarrStoreOpts, _url: Union[str, None] = None, ): """ @@ -76,17 +76,17 @@ def close(self): def from_file( hdf5_file_name_or_url: str, *, - opts: LindiH5StoreOpts = LindiH5StoreOpts(), + opts: LindiH5ZarrStoreOpts = LindiH5ZarrStoreOpts(), url: Union[str, None] = None, ): """ - Create a LindiH5Store from a file or url. + Create a LindiH5ZarrStore from a file or url. Parameters ---------- hdf5_file_name_or_url : str The name of the HDF5 file or a URL to the HDF5 file. - opts : LindiH5StoreOpts + opts : LindiH5ZarrStoreOpts Options for the store. url : str or None If hdf5_file_name_or_url is a local file name, then this can @@ -100,10 +100,10 @@ def from_file( "http://" ) or hdf5_file_name_or_url.startswith("https://"): remf = remfile.File(hdf5_file_name_or_url, verbose=False) - return LindiH5Store(_file=remf, _url=hdf5_file_name_or_url, _opts=opts) + return LindiH5ZarrStore(_file=remf, _url=hdf5_file_name_or_url, _opts=opts) else: f = open(hdf5_file_name_or_url, "rb") - return LindiH5Store(_file=f, _url=url, _opts=opts) + return LindiH5ZarrStore(_file=f, _url=url, _opts=opts) def __getitem__(self, key): """Get an item from the store (required by base class).""" @@ -397,7 +397,7 @@ def listdir(self, path: str = "") -> List[str]: def to_file(self, file_name: str, *, file_type: Literal["zarr.json"] = "zarr.json"): """Write a reference file system cooresponding to this store to a file. - This can then be loaded using LindiClient.from_file(fname) + This can then be loaded using LindiH5pyFile.from_file(fname) """ if file_type != "zarr.json": raise Exception(f"Unsupported file type: {file_type}") @@ -410,7 +410,7 @@ def to_reference_file_system(self) -> dict: """Create a reference file system cooresponding to this store. This can then be loaded using - LindiClient.from_reference_file_system(obj) + LindiH5pyFile.from_reference_file_system(obj) """ if self._h5f is None: raise Exception("Store is closed") diff --git a/lindi/LindiH5ZarrStore/__init__.py b/lindi/LindiH5ZarrStore/__init__.py new file mode 100644 index 0000000..c7bdb5b --- /dev/null +++ b/lindi/LindiH5ZarrStore/__init__.py @@ -0,0 +1 @@ +from .LindiH5ZarrStore import LindiH5ZarrStore, LindiH5ZarrStoreOpts # noqa: F401 diff --git a/lindi/LindiH5Store/_h5_attr_to_zarr_attr.py b/lindi/LindiH5ZarrStore/_h5_attr_to_zarr_attr.py similarity index 98% rename from lindi/LindiH5Store/_h5_attr_to_zarr_attr.py rename to lindi/LindiH5ZarrStore/_h5_attr_to_zarr_attr.py index 533f04c..3e5c306 100644 --- a/lindi/LindiH5Store/_h5_attr_to_zarr_attr.py +++ b/lindi/LindiH5ZarrStore/_h5_attr_to_zarr_attr.py @@ -74,7 +74,7 @@ def _h5_ref_to_zarr_attr(ref: h5py.Reference, *, label: str = '', h5f: h5py.File object_id = dref_obj.attrs.get("object_id", None) # Here we assume that the file has a top-level attribute called "object_id". - # This will be the case for files created by the LindiH5Store class. + # This will be the case for files created by the LindiH5ZarrStore class. file_object_id = h5f.attrs.get("object_id", None) # See https://hdmf-zarr.readthedocs.io/en/latest/storage.html#storing-object-references-in-attributes diff --git a/lindi/LindiH5Store/_h5_filters_to_codecs.py b/lindi/LindiH5ZarrStore/_h5_filters_to_codecs.py similarity index 100% rename from lindi/LindiH5Store/_h5_filters_to_codecs.py rename to lindi/LindiH5ZarrStore/_h5_filters_to_codecs.py diff --git a/lindi/LindiH5Store/_util.py b/lindi/LindiH5ZarrStore/_util.py similarity index 100% rename from lindi/LindiH5Store/_util.py rename to lindi/LindiH5ZarrStore/_util.py diff --git a/lindi/LindiH5Store/_zarr_info_for_h5_dataset.py b/lindi/LindiH5ZarrStore/_zarr_info_for_h5_dataset.py similarity index 99% rename from lindi/LindiH5Store/_zarr_info_for_h5_dataset.py rename to lindi/LindiH5ZarrStore/_zarr_info_for_h5_dataset.py index 3d1a4b9..c8ed2ef 100644 --- a/lindi/LindiH5Store/_zarr_info_for_h5_dataset.py +++ b/lindi/LindiH5ZarrStore/_zarr_info_for_h5_dataset.py @@ -24,7 +24,7 @@ class ZarrInfoForH5Dataset: def _zarr_info_for_h5_dataset(h5_dataset: h5py.Dataset) -> ZarrInfoForH5Dataset: """Get the information needed to create a zarr dataset from an h5py dataset. - This is the main workhorse function for LindiH5Store. It takes an h5py + This is the main workhorse function for LindiH5ZarrStore. It takes an h5py dataset and returns a ZarrInfoForH5Dataset object. It handles the following cases: diff --git a/lindi/LindiClient/LindiAttributes.py b/lindi/LindiH5pyFile/LindiAttributes.py similarity index 100% rename from lindi/LindiClient/LindiAttributes.py rename to lindi/LindiH5pyFile/LindiAttributes.py diff --git a/lindi/LindiClient/LindiDataset.py b/lindi/LindiH5pyFile/LindiDataset.py similarity index 100% rename from lindi/LindiClient/LindiDataset.py rename to lindi/LindiH5pyFile/LindiDataset.py diff --git a/lindi/LindiClient/LindiGroup.py b/lindi/LindiH5pyFile/LindiGroup.py similarity index 100% rename from lindi/LindiClient/LindiGroup.py rename to lindi/LindiH5pyFile/LindiGroup.py diff --git a/lindi/LindiClient/LindiClient.py b/lindi/LindiH5pyFile/LindiH5pyFile.py similarity index 86% rename from lindi/LindiClient/LindiClient.py rename to lindi/LindiH5pyFile/LindiH5pyFile.py index e745dd5..313b43f 100644 --- a/lindi/LindiClient/LindiClient.py +++ b/lindi/LindiH5pyFile/LindiH5pyFile.py @@ -11,7 +11,7 @@ from .LindiReference import LindiReference -class LindiClient(LindiGroup): +class LindiH5pyFile(LindiGroup): def __init__( self, *, @@ -25,15 +25,15 @@ def filename(self): return '' @staticmethod - def from_zarr_store(zarr_store: Union[Store, FSMap]) -> "LindiClient": + def from_zarr_store(zarr_store: Union[Store, FSMap]) -> "LindiH5pyFile": zarr_group = zarr.open(store=zarr_store, mode="r") assert isinstance(zarr_group, zarr.Group) - return LindiClient.from_zarr_group(zarr_group) + return LindiH5pyFile.from_zarr_group(zarr_group) @staticmethod def from_file( json_file: str, file_type: Literal["zarr.json"] = "zarr.json" - ) -> "LindiClient": + ) -> "LindiH5pyFile": if file_type == "zarr.json": if json_file.startswith("http") or json_file.startswith("https"): with tempfile.TemporaryDirectory() as tmpdir: @@ -41,22 +41,22 @@ def from_file( _download_file(json_file, filename) with open(filename, "r") as f: data = json.load(f) - return LindiClient.from_reference_file_system(data) + return LindiH5pyFile.from_reference_file_system(data) else: with open(json_file, "r") as f: data = json.load(f) - return LindiClient.from_reference_file_system(data) + return LindiH5pyFile.from_reference_file_system(data) else: raise ValueError(f"Unknown file_type: {file_type}") @staticmethod - def from_zarr_group(zarr_group: zarr.Group) -> "LindiClient": - return LindiClient(_zarr_group=zarr_group) + def from_zarr_group(zarr_group: zarr.Group) -> "LindiH5pyFile": + return LindiH5pyFile(_zarr_group=zarr_group) @staticmethod - def from_reference_file_system(data: dict) -> "LindiClient": + def from_reference_file_system(data: dict) -> "LindiH5pyFile": fs = ReferenceFileSystem(data).get_mapper(root="") - return LindiClient.from_zarr_store(fs) + return LindiH5pyFile.from_zarr_store(fs) def get(self, key, default=None, getlink: bool = False): try: @@ -95,7 +95,7 @@ def __getitem__(self, key): # type: ignore raise Exception(f'Mismatch in object_id: "{key._object_id}" and "{target.attrs.get("object_id")}"') return target else: - raise Exception(f'Cannot use key "{key}" of type "{type(key)}" to index into a LindiClient') + raise Exception(f'Cannot use key "{key}" of type "{type(key)}" to index into a LindiH5pyFile') def _download_file(url: str, filename: str) -> None: diff --git a/lindi/LindiClient/LindiReference.py b/lindi/LindiH5pyFile/LindiReference.py similarity index 100% rename from lindi/LindiClient/LindiReference.py rename to lindi/LindiH5pyFile/LindiReference.py diff --git a/lindi/LindiClient/__init__.py b/lindi/LindiH5pyFile/__init__.py similarity index 79% rename from lindi/LindiClient/__init__.py rename to lindi/LindiH5pyFile/__init__.py index bd5d2d6..e954c7c 100644 --- a/lindi/LindiClient/__init__.py +++ b/lindi/LindiH5pyFile/__init__.py @@ -1,4 +1,4 @@ -from .LindiClient import LindiClient # noqa: F401 +from .LindiH5pyFile import LindiH5pyFile # noqa: F401 from .LindiGroup import LindiGroup # noqa: F401 from .LindiDataset import LindiDataset # noqa: F401 from .LindiAttributes import LindiAttributes # noqa: F401 diff --git a/lindi/__init__.py b/lindi/__init__.py index 7d82117..30e4300 100644 --- a/lindi/__init__.py +++ b/lindi/__init__.py @@ -1,2 +1,2 @@ -from .LindiClient import LindiClient, LindiGroup, LindiDataset, LindiAttributes, LindiReference # noqa: F401 -from .LindiH5Store import LindiH5Store, LindiH5StoreOpts # noqa: F401 +from .LindiH5pyFile import LindiH5pyFile, LindiGroup, LindiDataset, LindiAttributes, LindiReference # noqa: F401 +from .LindiH5ZarrStore import LindiH5ZarrStore, LindiH5ZarrStoreOpts # noqa: F401 diff --git a/tests/test_core.py b/tests/test_core.py index 1adb42a..ce4b65b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,7 +1,7 @@ import numpy as np import h5py import tempfile -from lindi import LindiH5Store, LindiClient, LindiDataset, LindiGroup, LindiReference +from lindi import LindiH5ZarrStore, LindiH5pyFile, LindiDataset, LindiGroup, LindiReference import pytest @@ -13,11 +13,11 @@ def test_scalar_datasets(): with h5py.File(filename, "w") as f: ds = f.create_dataset("X", data=val) ds.attrs["foo"] = "bar" - with LindiH5Store.from_file( + with LindiH5ZarrStore.from_file( filename, url=filename ) as store: # set url so that a reference file system can be created rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) h5f = h5py.File(filename, "r") X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) @@ -29,7 +29,7 @@ def test_scalar_datasets(): assert '.zgroup' in store assert '.zarray' not in rfs['refs'] assert '.zarray' not in store - assert '.zattrs' in store # it's in the store but not in the ref file system -- see notes in LindiH5Store source code + assert '.zattrs' in store # it's in the store but not in the ref file system -- see notes in LindiH5ZarrStore source code assert '.zattrs' not in rfs['refs'] assert 'X/.zgroup' not in store assert 'X/.zattrs' in store # foo is set to bar @@ -48,11 +48,11 @@ def test_numpy_arrays(): filename = f"{tmpdir}/test.h5" with h5py.File(filename, "w") as f: f.create_dataset("X", data=array, chunks=chunks) - with LindiH5Store.from_file( + with LindiH5ZarrStore.from_file( filename, url=filename ) as store: # set url so that a reference file system can be created rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) h5f = h5py.File(filename, "r") X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) @@ -85,9 +85,9 @@ def test_numpy_array_of_strings(): with h5py.File(filename, "w") as f: f.create_dataset("X", data=["abc", "def", "ghi"]) h5f = h5py.File(filename, "r") - with LindiH5Store.from_file(filename, url=filename) as store: + with LindiH5ZarrStore.from_file(filename, url=filename) as store: rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) X2 = client["X"] @@ -107,9 +107,9 @@ def test_compound_dtype(): dt = np.dtype([("x", "i4"), ("y", "f8")]) f.create_dataset("X", data=[(1, 3.14), (2, 6.28)], dtype=dt) h5f = h5py.File(filename, "r") - store = LindiH5Store.from_file(filename, url=filename) + store = LindiH5ZarrStore.from_file(filename, url=filename) rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) X2 = client["X"] @@ -146,9 +146,9 @@ def test_attributes(): f["group"].attrs["foo"] = "bar2" f["group"].attrs["baz"] = 3.15 h5f = h5py.File(filename, "r") - with LindiH5Store.from_file(filename, url=filename) as store: + with LindiH5ZarrStore.from_file(filename, url=filename) as store: rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) @@ -209,9 +209,9 @@ def test_nan_inf_attr(): f["X"].attrs["inf"] = np.inf f["X"].attrs["ninf"] = -np.inf h5f = h5py.File(filename, "r") - with LindiH5Store.from_file(filename, url=filename) as store: + with LindiH5ZarrStore.from_file(filename, url=filename) as store: rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) @@ -232,9 +232,9 @@ def test_reference_attributes(): Y_ds = f.create_dataset("Y", data=[4, 5, 6]) X_ds.attrs["ref"] = Y_ds.ref h5f = h5py.File(filename, "r") - with LindiH5Store.from_file(filename, url=filename) as store: + with LindiH5ZarrStore.from_file(filename, url=filename) as store: rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) @@ -263,9 +263,9 @@ def test_reference_in_compound_dtype(): Y_ds = f.create_dataset("Y", data=[1, 2, 3]) f.create_dataset("X", data=[(1, Y_ds.ref), (2, Y_ds.ref)], dtype=compound_dtype) h5f = h5py.File(filename, "r") - with LindiH5Store.from_file(filename, url=filename) as store: + with LindiH5ZarrStore.from_file(filename, url=filename) as store: rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) X1 = h5f["X"] assert isinstance(X1, h5py.Dataset) diff --git a/tests/test_with_real_data.py b/tests/test_with_real_data.py index ff68502..ca277a1 100644 --- a/tests/test_with_real_data.py +++ b/tests/test_with_real_data.py @@ -3,7 +3,7 @@ import zarr import h5py import remfile -from lindi import LindiH5Store, LindiClient +from lindi import LindiH5ZarrStore, LindiH5pyFile import lindi import pytest @@ -281,16 +281,16 @@ def test_with_real_data(): h5_url = example["h5_url"] print(f"Running comparison for {h5_url}") h5f = h5py.File(remfile.File(h5_url), "r") - with LindiH5Store.from_file(h5_url) as store: + with LindiH5ZarrStore.from_file(h5_url) as store: rfs = store.to_reference_file_system() - client = LindiClient.from_reference_file_system(rfs) + client = LindiH5pyFile.from_reference_file_system(rfs) # visit the items in the h5py file and compare them to the zarr file _hdf5_visit_items(h5f, lambda key, item: _compare_item(item, client[key])) with tempfile.TemporaryDirectory() as tmpdir: store.to_file(f"{tmpdir}/example.zarr.json") - LindiClient.from_file(f"{tmpdir}/example.zarr.json") + LindiH5pyFile.from_file(f"{tmpdir}/example.zarr.json") top_level_keys = [k for k in h5f.keys()] top_level_keys_2 = store.listdir()