From 0106750d860fc9701dbaef7e0c5cde40235c2f7a Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 28 Feb 2025 18:06:50 -0500 Subject: [PATCH 1/6] Mark hanging test as xfail --- .github/workflows/main.yml | 1 + .github/workflows/min-deps.yml | 1 + .github/workflows/upstream.yml | 1 + virtualizarr/tests/test_readers/test_kerchunk.py | 3 +++ 4 files changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b50d6f56..23d9b2ea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,6 +29,7 @@ jobs: strategy: matrix: python-version: ["3.11", "3.12"] + timeout-minutes: 30 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/min-deps.yml b/.github/workflows/min-deps.yml index c236a9ef..c1e13c78 100644 --- a/.github/workflows/min-deps.yml +++ b/.github/workflows/min-deps.yml @@ -29,6 +29,7 @@ jobs: strategy: matrix: python-version: ["3.12"] + timeout-minutes: 30 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 74867ea5..e07e5163 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -29,6 +29,7 @@ jobs: strategy: matrix: python-version: ["3.12"] + timeout-minutes: 30 steps: - uses: actions/checkout@v4 diff --git a/virtualizarr/tests/test_readers/test_kerchunk.py b/virtualizarr/tests/test_readers/test_kerchunk.py index 83f7999d..ab86843b 100644 --- a/virtualizarr/tests/test_readers/test_kerchunk.py +++ b/virtualizarr/tests/test_readers/test_kerchunk.py @@ -233,6 +233,9 @@ def test_open_virtual_dataset_existing_kerchunk_refs( @requires_kerchunk +@pytest.mark.xfail( + reason="Test hangs after https://github.com/zarr-developers/VirtualiZarr/pull/420" +) def test_notimplemented_read_inline_refs(tmp_path, netcdf4_inlined_ref): # For now, we raise a NotImplementedError if we read existing references that have inlined data # https://github.com/zarr-developers/VirtualiZarr/pull/251#pullrequestreview-2361916932 From 479ad527f93016bf3b33554be583c55f21ecd66f Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 28 Feb 2025 19:36:51 -0500 Subject: [PATCH 2/6] Use distinct file paths --- virtualizarr/tests/test_readers/conftest.py | 18 +++++++++++------- .../tests/test_readers/test_kerchunk.py | 7 ++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/virtualizarr/tests/test_readers/conftest.py b/virtualizarr/tests/test_readers/conftest.py index 4884db4a..a38f9783 100644 --- a/virtualizarr/tests/test_readers/conftest.py +++ b/virtualizarr/tests/test_readers/conftest.py @@ -372,18 +372,22 @@ def scalar_fill_value_hdf5_file(tmpdir): compound_fill = (-9999, -9999.0) fill_values = [ - {"fill_value": -9999, "data": np.random.randint(0, 10, size=(5))}, - {"fill_value": -9999.0, "data": np.random.random(5)}, - {"fill_value": np.nan, "data": np.random.random(5)}, - {"fill_value": False, "data": np.array([True, False, False, True, True])}, - {"fill_value": "NaN", "data": np.array(["three"], dtype="S10")}, - {"fill_value": compound_fill, "data": compound_data}, + {"label": "int", "fill_value": -9999, "data": np.random.randint(0, 10, size=(5))}, + {"label": "float", "fill_value": -9999.0, "data": np.random.random(5)}, + {"label": "npNan", "fill_value": np.nan, "data": np.random.random(5)}, + { + "label": "False", + "fill_value": False, + "data": np.array([True, False, False, True, True]), + }, + {"label": "NaN", "fill_value": "NaN", "data": np.array(["three"], dtype="S10")}, + {"label": "compound", "fill_value": compound_fill, "data": compound_data}, ] @pytest.fixture(params=fill_values) def cf_fill_value_hdf5_file(tmpdir, request): - filepath = f"{tmpdir}/cf_fill_value.nc" + filepath = f"{tmpdir}/cf_fill_value_{request.param['label']}.nc" f = h5py.File(filepath, "w") dset = f.create_dataset(name="data", data=request.param["data"], chunks=True) dim_scale = f.create_dataset( diff --git a/virtualizarr/tests/test_readers/test_kerchunk.py b/virtualizarr/tests/test_readers/test_kerchunk.py index ab86843b..165a563b 100644 --- a/virtualizarr/tests/test_readers/test_kerchunk.py +++ b/virtualizarr/tests/test_readers/test_kerchunk.py @@ -201,7 +201,7 @@ def test_open_virtual_dataset_existing_kerchunk_refs( # Test valid json and parquet reference formats if reference_format == "json": - ref_filepath = tmp_path / "ref.json" + ref_filepath = tmp_path / "ref1.json" import ujson @@ -233,14 +233,11 @@ def test_open_virtual_dataset_existing_kerchunk_refs( @requires_kerchunk -@pytest.mark.xfail( - reason="Test hangs after https://github.com/zarr-developers/VirtualiZarr/pull/420" -) def test_notimplemented_read_inline_refs(tmp_path, netcdf4_inlined_ref): # For now, we raise a NotImplementedError if we read existing references that have inlined data # https://github.com/zarr-developers/VirtualiZarr/pull/251#pullrequestreview-2361916932 - ref_filepath = tmp_path / "ref.json" + ref_filepath = tmp_path / "ref2.json" import ujson From 9dbd713d7afd8371e234bdfea1b63b7bfa0739cc Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 28 Feb 2025 19:45:30 -0500 Subject: [PATCH 3/6] Disable fsspec caching in fixtures --- conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 0781c37e..2e8fc73a 100644 --- a/conftest.py +++ b/conftest.py @@ -103,7 +103,11 @@ def netcdf4_virtual_dataset(netcdf4_file): def netcdf4_inlined_ref(netcdf4_file): from kerchunk.hdf import SingleHdf5ToZarr - return SingleHdf5ToZarr(netcdf4_file, inline_threshold=1000).translate() + return SingleHdf5ToZarr( + netcdf4_file, + inline_threshold=1000, + storage_options={"use_listings_cache": False}, + ).translate() @pytest.fixture From a4fdf74cab9699b732e172f7c77d7defd9b72a95 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 28 Feb 2025 19:48:56 -0500 Subject: [PATCH 4/6] Update virtualizarr/tests/test_readers/conftest.py Co-authored-by: Chuck Daniels --- virtualizarr/tests/test_readers/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_readers/conftest.py b/virtualizarr/tests/test_readers/conftest.py index a38f9783..f60d970e 100644 --- a/virtualizarr/tests/test_readers/conftest.py +++ b/virtualizarr/tests/test_readers/conftest.py @@ -387,7 +387,7 @@ def scalar_fill_value_hdf5_file(tmpdir): @pytest.fixture(params=fill_values) def cf_fill_value_hdf5_file(tmpdir, request): - filepath = f"{tmpdir}/cf_fill_value_{request.param['label']}.nc" + filepath = tmp_path / f"cf_fill_value_{request.param['label']}.nc" f = h5py.File(filepath, "w") dset = f.create_dataset(name="data", data=request.param["data"], chunks=True) dim_scale = f.create_dataset( From 41d114c3f0431439ca2c83594e28984dbce86286 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 28 Feb 2025 19:52:48 -0500 Subject: [PATCH 5/6] Fix typo --- virtualizarr/tests/test_readers/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_readers/conftest.py b/virtualizarr/tests/test_readers/conftest.py index f60d970e..9f3e20af 100644 --- a/virtualizarr/tests/test_readers/conftest.py +++ b/virtualizarr/tests/test_readers/conftest.py @@ -387,7 +387,7 @@ def scalar_fill_value_hdf5_file(tmpdir): @pytest.fixture(params=fill_values) def cf_fill_value_hdf5_file(tmpdir, request): - filepath = tmp_path / f"cf_fill_value_{request.param['label']}.nc" + filepath = tmpdir / f"cf_fill_value_{request.param['label']}.nc" f = h5py.File(filepath, "w") dset = f.create_dataset(name="data", data=request.param["data"], chunks=True) dim_scale = f.create_dataset( From 133eea42229966d3ac9af94f5ef88f8069ca4e91 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 28 Feb 2025 19:54:21 -0500 Subject: [PATCH 6/6] Revert naming change --- virtualizarr/tests/test_readers/test_kerchunk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virtualizarr/tests/test_readers/test_kerchunk.py b/virtualizarr/tests/test_readers/test_kerchunk.py index 165a563b..83f7999d 100644 --- a/virtualizarr/tests/test_readers/test_kerchunk.py +++ b/virtualizarr/tests/test_readers/test_kerchunk.py @@ -201,7 +201,7 @@ def test_open_virtual_dataset_existing_kerchunk_refs( # Test valid json and parquet reference formats if reference_format == "json": - ref_filepath = tmp_path / "ref1.json" + ref_filepath = tmp_path / "ref.json" import ujson @@ -237,7 +237,7 @@ def test_notimplemented_read_inline_refs(tmp_path, netcdf4_inlined_ref): # For now, we raise a NotImplementedError if we read existing references that have inlined data # https://github.com/zarr-developers/VirtualiZarr/pull/251#pullrequestreview-2361916932 - ref_filepath = tmp_path / "ref2.json" + ref_filepath = tmp_path / "ref.json" import ujson