pydata · dcherian · Mar 26, 2025 · Mar 21, 2025 · Mar 21, 2025 · Mar 21, 2025
diff --git a/doc/user-guide/io.rst b/doc/user-guide/io.rst
@@ -1021,6 +1021,21 @@ reads. Because this fall-back option is so much slower, xarray issues a
        instead of falling back to try reading non-consolidated metadata.
 
 
+Fill Values
+~~~~~~~~~~~
+
+Zarr arrays have a ``fill_value`` that is used for chunks that were never written to disk.
+For the Zarr version 2 format, Xarray will set ``fill_value`` to be equal to the CF/NetCDF ``"_FillValue"``.
+This is ``np.nan`` by default for floats, and unset otherwise. Note that the Zarr library will set a
+default ``fill_value`` if not specified (usually ``0``).
+
+For the Zarr version 3 format, ``_FillValue`` and ```fill_value`` are decoupled.
+So you can set ``fill_value`` in ``encoding`` as usual.
+
+Note that at read-time, you can control whether ``_FillValue`` is masked using the
+``mask_and_scale`` kwarg.
+
+
 .. _io.kerchunk:
 
 Kerchunk

diff --git a/doc/whats-new.rst b/doc/whats-new.rst
@@ -22,6 +22,8 @@ v2025.03.1 (unreleased)
 New Features
 ~~~~~~~~~~~~
 
+- Allow setting a ``fill_value`` for Zarr format 3 arrays. Specify ``fill_value`` in ``encoding`` as usual.
+  (:issue:`10064`). By `Deepak Cherian <https://github.com/dcherian>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~

diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py
@@ -426,6 +426,7 @@ def extract_zarr_variable_encoding(
     raise_on_invalid=False,
     name=None,
     *,
+    zarr_format: ZarrFormat,
     safe_chunks=True,
     region=None,
     mode=None,
@@ -443,6 +444,7 @@ def extract_zarr_variable_encoding(
     region: tuple[slice, ...], optional
     mode: str, optional
     shape: tuple[int, ...], optional
+    zarr_format: Literal[2,3]
     Returns
     -------
     encoding : dict
@@ -463,16 +465,23 @@ def extract_zarr_variable_encoding(
         "cache_metadata",
         "write_empty_chunks",
     }
+    if zarr_format == 3:
+        valid_encodings.add("fill_value")
 
     for k in safe_to_drop:
         if k in encoding:
             del encoding[k]
 
     if raise_on_invalid:
         invalid = [k for k in encoding if k not in valid_encodings]
+        if "fill_value" in invalid and zarr_format == 2:
+            msg = " Use `_FillValue` to set the Zarr array `fill_value`"
+        else:
+            msg = ""
+
         if invalid:
             raise ValueError(
-                f"unexpected encoding parameters for zarr backend:  {invalid!r}"
+                f"unexpected encoding parameters for zarr backend:  {invalid!r}." + msg
             )
     else:
         for k in list(encoding):
@@ -1147,7 +1156,7 @@ def set_variables(self, variables, check_encoding_set, writer, unlimited_dims=No
             if self._use_zarr_fill_value_as_mask:
                 fill_value = attrs.pop("_FillValue", None)
             else:
-                fill_value = None
+                fill_value = v.encoding.pop("fill_value", None)
                 if "_FillValue" in attrs:
                     # replace with encoded fill value
                     fv = attrs.pop("_FillValue")
@@ -1198,6 +1207,7 @@ def set_variables(self, variables, check_encoding_set, writer, unlimited_dims=No
                 region=region,
                 mode=self._mode,
                 shape=zarr_shape,
+                zarr_format=self.zarr_group.metadata.zarr_format,
             )
 
             if self._mode == "w" or name not in existing_keys:

diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
@@ -13,6 +13,7 @@
 import tempfile
 import uuid
 import warnings
+from collections import ChainMap
 from collections.abc import Generator, Iterator, Mapping
 from contextlib import ExitStack
 from io import BytesIO
@@ -661,7 +662,7 @@
            {"foo": ("x", [0, 1])}, {"x": [2, 3], "y": ("a", [42]), "z": ("x", [4, 5])}
        )

        with self.roundtrip(original) as actual:
            assert_identical(original, actual)

        original["foo"].encoding["coordinates"] = "y"
@@ -676,7 +677,7 @@
        original = Dataset(
            {"foo": ("x", [0, 1])}, {"x": [2, 3], "y": ("a", [42]), "z": ("x", [4, 5])}
        )
        with self.roundtrip(original) as actual:
            assert_identical(original, actual)

        # test that global "coordinates" is as expected
@@ -693,13 +694,13 @@
        original = Dataset(coords={"x": 0, "y z": 1})
        expected = Dataset({"y z": 1}, {"x": 0})
        with pytest.warns(SerializationWarning):
            with self.roundtrip(original) as actual:
                assert_identical(expected, actual)

    def test_roundtrip_boolean_dtype(self) -> None:
        original = create_boolean_data()
        assert original["x"].dtype == "bool"
        with self.roundtrip(original) as actual:
            assert_identical(original, actual)
            assert actual["x"].dtype == "bool"
            # this checks for preserving dtype during second roundtrip
@@ -3343,6 +3344,61 @@
             observed_keys_2 = sorted(zstore_mut.array_keys())
             assert observed_keys_2 == sorted(array_keys + [new_key])
 
+    @requires_dask
+    @pytest.mark.parametrize("dtype", [int, float])
+    def test_zarr_fill_value_setting(self, dtype):
+        # When zarr_format=2, _FillValue sets fill_value
+        # When zarr_format=3, fill_value is set independently
+        # We test this by writing a dask array with compute=False,
+        # on read we should receive chunks filled with `fill_value`
+        fv = -1
+        ds = xr.Dataset(
+            {"foo": ("x", dask.array.from_array(np.array([0, 0, 0], dtype=dtype)))}
+        )
+        expected = xr.Dataset({"foo": ("x", [fv] * 3)})
+
+        zarr_format_2 = (
+            has_zarr_v3 and zarr.config.get("default_zarr_format") == 2
+        ) or not has_zarr_v3
+        if zarr_format_2:
+            attr = "_FillValue"
+            expected.foo.attrs[attr] = fv
+        else:
+            attr = "fill_value"
+            if dtype is float:
+                # for floats, Xarray inserts a default `np.nan`
+                expected.foo.attrs["_FillValue"] = np.nan
 fill_value = None 
 if "_FillValue" in attrs: 
     # replace with encoded fill value 
     fv = attrs.pop("_FillValue") 
     if fv is not None: 
         attrs["_FillValue"] = FillValueCoder.encode(fv, dtype) 
 fill_value = None 
 if "_FillValue" in attrs: 
     # replace with encoded fill value 
     fv = attrs.pop("_FillValue") 
     if fv is not None: 
         attrs["_FillValue"] = FillValueCoder.encode(fv, dtype) 
+
+        open_kwargs = {"mask_and_scale": False, "consolidated": False}
+        save_kwargs = dict(compute=False, consolidated=False)
+        with self.roundtrip(
+            ds,
+            save_kwargs=ChainMap(save_kwargs, dict(encoding={"foo": {attr: fv}})),
+            open_kwargs=open_kwargs,
+        ) as actual:
+            assert_identical(actual, expected)
+
+        ds.foo.encoding[attr] = fv
+        with self.roundtrip(
+            ds, save_kwargs=save_kwargs, open_kwargs=open_kwargs
+        ) as actual:
+            assert_identical(actual, expected)
+
+        if zarr_format_2:
+            ds = ds.drop_encoding()
+            with pytest.raises(ValueError, match="_FillValue"):
+                with self.roundtrip(
+                    ds,
+                    save_kwargs=ChainMap(
+                        save_kwargs, dict(encoding={"foo": {"fill_value": fv}})
+                    ),
+                    open_kwargs=open_kwargs,
+                ):
+                    pass
+            # TODO: this doesn't fail because of the
+            # ``raise_on_invalid=vn in check_encoding_set`` line in zarr.py
+            # ds.foo.encoding["fill_value"] = fv
+
 
 @requires_zarr
 @pytest.mark.skipif(
@@ -3720,7 +3776,7 @@
            )

        with self.create_zarr_target() as store, patched as mock:
            ds.to_zarr(store, mode="w")

            # We expect this to request array metadata information, so call_count should be == 1,
            xrds = xr.open_zarr(store)
@@ -5827,23 +5883,23 @@
 @requires_zarr
 def test_extract_zarr_variable_encoding() -> None:
     var = xr.Variable("x", [1, 2])
-    actual = backends.zarr.extract_zarr_variable_encoding(var)
+    actual = backends.zarr.extract_zarr_variable_encoding(var, zarr_format=3)
     assert "chunks" in actual
     assert actual["chunks"] == ("auto" if has_zarr_v3 else None)
 
     var = xr.Variable("x", [1, 2], encoding={"chunks": (1,)})
-    actual = backends.zarr.extract_zarr_variable_encoding(var)
+    actual = backends.zarr.extract_zarr_variable_encoding(var, zarr_format=3)
     assert actual["chunks"] == (1,)
 
     # does not raise on invalid
     var = xr.Variable("x", [1, 2], encoding={"foo": (1,)})
-    actual = backends.zarr.extract_zarr_variable_encoding(var)
+    actual = backends.zarr.extract_zarr_variable_encoding(var, zarr_format=3)
 
     # raises on invalid
     var = xr.Variable("x", [1, 2], encoding={"foo": (1,)})
     with pytest.raises(ValueError, match=r"unexpected encoding parameters"):
         actual = backends.zarr.extract_zarr_variable_encoding(
-            var, raise_on_invalid=True
+            var, raise_on_invalid=True, zarr_format=3
         )
 
 
@@ -5862,7 +5918,7 @@

    m = fsspec.filesystem("memory")
    mm = m.get_mapper("out1.zarr")
    ds.to_zarr(mm)  # old interface
    ds0 = ds.copy()
    # pd.to_timedelta returns ns-precision, but the example data is in second precision
    # so we need to fix this
@@ -5934,7 +5990,7 @@
        }
    )
    ds["test"].encoding["chunks"] = encoded_chunks
    ds.to_zarr(tmp_path / "test.zarr")

    with dask.config.set({"array.chunk-size": "1MiB"}):
        expected = ds.chunk(chunks)
@@ -6148,7 +6204,7 @@
 def test_zarr_closing_internal_zip_store():
    store_name = "tmp.zarr.zip"
    original_da = DataArray(np.arange(12).reshape((3, 4)))
    original_da.to_zarr(store_name, mode="w")

    with open_dataarray(store_name, engine="zarr") as loaded_da:
        assert_identical(original_da, loaded_da)
@@ -6199,7 +6255,7 @@
        ],
    )
    def test_zarr_region_auto(self, region):
        with self.create() as (target, ds):
            ds_region = 1 + ds.isel(x=slice(2, 4), y=slice(6, 8))
            self.save(target, ds_region, region=region)
            ds_updated = xr.open_zarr(target)
@@ -6209,7 +6265,7 @@
            assert_identical(ds_updated, expected)

    def test_zarr_region_auto_noncontiguous(self):
        with self.create() as (target, ds):
            with pytest.raises(ValueError):
                self.save(target, ds.isel(x=[0, 2, 3], y=[5, 6]), region="auto")

@@ -6222,7 +6278,7 @@
        region: Mapping[str, slice] | Literal["auto"]
        region_slice = dict(x=slice(2, 4), y=slice(6, 8))

        with self.create() as (target, ds):
            ds_region = 1 + ds.isel(region_slice)
            for region in [region_slice, "auto"]:  # type: ignore[assignment]
                with patch.object(
@@ -6241,7 +6297,7 @@
                        assert "y" not in written_variables

    def test_zarr_region_append(self):
        with self.create() as (target, ds):
            x_new = np.arange(40, 70, 10)
            data_new = np.ones((3, 10))
            ds_new = xr.Dataset(
@@ -6261,7 +6317,7 @@
                self.save(target, ds_new, mode="a", append_dim="x", region="auto")

    def test_zarr_region(self):
        with self.create() as (target, ds):
            ds_transposed = ds.transpose("y", "x")
            ds_region = 1 + ds_transposed.isel(x=[0], y=[0])
            self.save(target, ds_region, region={"x": slice(0, 1), "y": slice(0, 1)})
@@ -6281,7 +6337,7 @@
        )

        with self.create_zarr_target() as target:
            self.save(target, ds.chunk(5), compute=False, mode="w")
            with pytest.raises(ValueError):
                for r in range(ds.sizes["a"]):
                    self.save(
@@ -6308,7 +6364,7 @@
        )

        with self.create_zarr_target() as target:
            self.save(target, da, mode="w", encoding={"foo": {"chunks": (5, 5, 1)}})

            with pytest.raises(ValueError, match="encoding was provided"):
                self.save(