diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 8bdf0f78eed..f45958f4a00 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -14,6 +14,8 @@ New Features ~~~~~~~~~~~~ - Expose :py:class:`~xarray.indexes.RangeIndex`, and :py:class:`~xarray.indexes.CoordinateTransformIndex` as public api under the ``xarray.indexes`` namespace. By `Deepak Cherian `_. +- Support zarr-python's new ``.supports_consolidated_metadata`` store property (:pull:`10457``). + by Tom Nicholas `_. - Better error messages when encoding data to be written to disk fails (:pull:`10464`). By `Stephan Hoyer `_ diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index fff07063964..48405b906cd 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1768,6 +1768,11 @@ def _get_open_params( else: missing_exc = zarr.errors.GroupNotFoundError + if _zarr_v3(): + # zarr 3.0.8 and earlier did not support this property - it was effectively assumed true + if not getattr(store, "supports_consolidated_metadata", True): + consolidated = consolidate_on_close = False + if consolidated in [None, True]: # open the root of the store, in case there is metadata consolidated there group = open_kwargs.pop("path") @@ -1825,6 +1830,7 @@ def _get_open_params( else: # this was the default for v2 and should apply to most existing Zarr data use_zarr_fill_value_as_mask = True + return ( zarr_group, consolidate_on_close, diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 2c1718cee7a..7bacaf58155 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -89,6 +89,7 @@ requires_scipy, requires_scipy_or_netCDF4, requires_zarr, + requires_zarr_v3, ) from xarray.tests.test_coding_times import ( _ALL_CALENDARS, @@ -117,6 +118,7 @@ if has_zarr_v3: from zarr.storage import MemoryStore as KVStore + from zarr.storage import WrapperStore ZARR_FORMATS = [2, 3] else: @@ -127,8 +129,11 @@ ) except ImportError: KVStore = None # type: ignore[assignment,misc,unused-ignore] + + WrapperStore = object # type: ignore[assignment,misc,unused-ignore] else: KVStore = None # type: ignore[assignment,misc,unused-ignore] + WrapperStore = object # type: ignore[assignment,misc,unused-ignore] ZARR_FORMATS = [] @@ -2400,12 +2405,13 @@ def test_read_non_consolidated_warning(self) -> None: self.save( expected, store_target=store, consolidated=False, **self.version_kwargs ) - with pytest.warns( - RuntimeWarning, - match="Failed to open Zarr store with consolidated", - ): - with xr.open_zarr(store, **self.version_kwargs) as ds: - assert_identical(ds, expected) + if getattr(store, "supports_consolidated_metadata", True): + with pytest.warns( + RuntimeWarning, + match="Failed to open Zarr store with consolidated", + ): + with xr.open_zarr(store, **self.version_kwargs) as ds: + assert_identical(ds, expected) def test_non_existent_store(self) -> None: with pytest.raises( @@ -3756,6 +3762,42 @@ def test_chunk_key_encoding_v2(self) -> None: assert actual["var1"].encoding["chunks"] == (2, 2) +class NoConsolidatedMetadataSupportStore(WrapperStore): + """ + Store that explicitly does not support consolidated metadata. + + Useful as a proxy for stores like Icechunk, see https://github.com/zarr-developers/zarr-python/pull/3119. + """ + + supports_consolidated_metadata = False + + def __init__( + self, + store, + *, + read_only: bool = False, + ) -> None: + self._store = store.with_read_only(read_only=read_only) + + def with_read_only( + self, read_only: bool = False + ) -> NoConsolidatedMetadataSupportStore: + return type(self)( + store=self._store, + read_only=read_only, + ) + + +@requires_zarr_v3 +class TestZarrNoConsolidatedMetadataSupport(ZarrBase): + @contextlib.contextmanager + def create_zarr_target(self): + # TODO the zarr version would need to be >3.08 for the supports_consolidated_metadata property to have any effect + yield NoConsolidatedMetadataSupportStore( + zarr.storage.MemoryStore({}, read_only=False) + ) + + @requires_zarr @pytest.mark.skipif( ON_WINDOWS, pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy