WIP: single_zarr_to_reference_file

kerchunk/tests/test_hdf.py

@@ -1,8 +1,10 @@
 import fsspec
+import fsspec.implementations.reference as reffs
 import os.path as osp
 
 import kerchunk.hdf
 import numpy as np
+import pandas as pd
 import pytest
 import xarray as xr
 import zarr
@@ -13,6 +15,24 @@
 here = osp.dirname(__file__)
 
 
+@pytest.fixture(scope="module")
+def ds():
+    ds = xr.Dataset(
+        {
+            "x": xr.DataArray(np.linspace(-np.pi, np.pi, 10), dims=["x"]),
+            "y": xr.DataArray(np.linspace(-np.pi / 2, np.pi / 2, 10), dims=["y"]),
+            "time": xr.DataArray(pd.date_range("2020", "2021"), dims=["time"]),
+        },
+    )
+    ds["temp"] = (
+        np.cos(ds.x)
+        * np.sin(ds.y)
+        * xr.ones_like(ds.time).astype("float")
+        * np.random.random(ds.time.shape)
+    )
+    return ds
+
+
 def test_single():
     """Test creating references for a single HDF file"""
     url = "s3://noaa-nwm-retro-v2.0-pds/full_physics/2017/201704010000.CHRTOUT_DOMAIN1.comp"
@@ -322,3 +342,11 @@ def test_embed():
         "0.004609216572327277",
         "0.01298182345556785",
     ]
+
+
+def test_hdf_to_reference_file(tmpdir, ds):
+    fn1 = f"{tmpdir}/test1.nc"
+    ds.to_netcdf(fn1)
+
+    out = reffs.LazyReferenceMapper.create(f"{tmpdir}/out.parq", record_size=1)
+    SingleHdf5ToZarr(h5f=fn1, out=out).translate()

kerchunk/tests/test_zarr.py

@@ -96,3 +96,11 @@ def test_zarr_json_dump_succeeds(tmpdir, ds):
         inline_threshold=0,
     ).translate()
     ujson.dumps(one)
+
+
+def test_zarr_to_reference_file(tmpdir, ds):
+    fn1 = f"{tmpdir}/test1.zarr"
+    ds.to_zarr(fn1)
+
+    out = reffs.LazyReferenceMapper.create(f"{tmpdir}/out.parq", record_size=1)
+    kerchunk.zarr.ZarrToZarr(url=fn1, **{"out": out}).translate()

kerchunk/zarr.py

@@ -49,9 +49,9 @@ def single_zarr(
     inline_threshold = inline or inline_threshold
     if inline_threshold:
         refs = do_inline(refs, inline_threshold, remote_options=storage_options)
+    refs = kerchunk.utils.consolidate(refs)
     if isinstance(refs, LazyReferenceMapper):
         refs.flush()
-    refs = kerchunk.utils.consolidate(refs)
     return refs