Skip to content

Allow setting (or skipping) new indexes in open_dataset #8051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 27 commits into from
Jul 8, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
c703ebc
add set_indexes parameter to open_dataset
benbovy Aug 7, 2023
6f54cd5
implement set_indexes in (zarr) backend store
benbovy Aug 7, 2023
f77aac7
Merge branch 'main' into backend-set-indexes
dcherian Nov 13, 2023
eae983b
Merge branch 'main' into backend-set-indexes
TomNicholas Feb 3, 2024
dfe6496
Merge branch 'main' into backend-set-indexes
keewis Jun 29, 2025
145ae1c
replace `set_indexes` with `create_default_indexes`
keewis Jun 30, 2025
192c367
make sure indexes set by the backend survive
keewis Jun 30, 2025
f5823a7
also add the parameter to `open_datatree`
keewis Jun 30, 2025
2ff8402
share the implementation of the default indexes creation
keewis Jun 30, 2025
82d629b
Merge branch 'main' into backend-set-indexes
keewis Jun 30, 2025
de5ce26
Merge branch 'main' into backend-set-indexes
keewis Jul 1, 2025
294b2f7
check that the store backend entrypoint does not create default indexes
keewis Jul 1, 2025
5c3a843
actually do not create default indexes in the backends
keewis Jul 1, 2025
08939de
rename the helper
keewis Jul 1, 2025
0f281b1
Merge branch 'main' into backend-set-indexes
dcherian Jul 2, 2025
4620490
Merge branch 'main' into backend-set-indexes
dcherian Jul 3, 2025
95dbf8e
move the handling of `create_default_indexes` up the call stack
keewis Jul 3, 2025
eb4f866
Merge branch 'main' into backend-set-indexes
keewis Jul 3, 2025
d7e6daa
what's new
keewis Jul 3, 2025
687f0c2
Merge branch 'main' into backend-set-indexes
benbovy Jul 8, 2025
3d483d3
Fix
dcherian Jul 8, 2025
741564e
fix again
dcherian Jul 8, 2025
8889eda
also create default indexes without chunks
keewis Jul 8, 2025
804db4c
also copy `_close`
keewis Jul 8, 2025
75c1dd6
reuse the code for copying `_close`
keewis Jul 8, 2025
a0d94fb
refactor
dcherian Jul 8, 2025
bbc263b
Merge branch 'main' into backend-set-indexes
dcherian Jul 8, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
check that the store backend entrypoint does not create default indexes
  • Loading branch information
keewis committed Jul 1, 2025
commit 294b2f712425ca9424a4d6ad62912d653087beb5
61 changes: 61 additions & 0 deletions xarray/tests/test_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
from xarray.coding.variables import SerializationWarning
from xarray.conventions import encode_dataset_coordinates
from xarray.core import indexing
from xarray.core.indexes import PandasIndex
from xarray.core.options import set_options
from xarray.core.utils import module_available
from xarray.namedarray.pycompat import array_type
Expand Down Expand Up @@ -2050,6 +2051,26 @@ def test_encoding_enum__error_multiple_variable_with_changing_enum(self):
with self.roundtrip(original):
pass

@pytest.mark.parametrize("create_default_indexes", [True, False])
def test_create_default_indexes(self, tmp_path, create_default_indexes) -> None:
store_path = tmp_path / "tmp.nc"
original_ds = xr.Dataset(
{"data": ("x", np.arange(3))}, coords={"x": [-1, 0, 1]}
)
original_ds.to_netcdf(store_path, engine=self.engine, mode="w")

with open_dataset(
store_path,
engine=self.engine,
create_default_indexes=create_default_indexes,
) as loaded_ds:
if create_default_indexes:
assert list(loaded_ds.xindexes) == ["x"] and isinstance(
loaded_ds.xindexes["x"], PandasIndex
)
else:
assert len(loaded_ds.xindexes) == 0


@requires_netCDF4
class TestNetCDF4Data(NetCDF4Base):
Expand Down Expand Up @@ -4009,6 +4030,26 @@ def test_pickle(self) -> None:
def test_pickle_dataarray(self) -> None:
pass

@pytest.mark.parametrize("create_default_indexes", [True, False])
def test_create_default_indexes(self, tmp_path, create_default_indexes) -> None:
store_path = tmp_path / "tmp.nc"
original_ds = xr.Dataset(
{"data": ("x", np.arange(3))}, coords={"x": [-1, 0, 1]}
)
original_ds.to_netcdf(store_path, engine=self.engine, mode="w")

with open_dataset(
store_path,
engine=self.engine,
create_default_indexes=create_default_indexes,
) as loaded_ds:
if create_default_indexes:
assert list(loaded_ds.xindexes) == ["x"] and isinstance(
loaded_ds.xindexes["x"], PandasIndex
)
else:
assert len(loaded_ds.xindexes) == 0


@requires_scipy
class TestScipyFilePath(CFEncodedBase, NetCDF3Only):
Expand Down Expand Up @@ -6378,6 +6419,26 @@ def test_zarr_closing_internal_zip_store():
assert_identical(original_da, loaded_da)


@requires_zarr
@pytest.mark.parametrize("create_default_indexes", [True, False])
def test_zarr_create_default_indexes(tmp_path, create_default_indexes) -> None:
from xarray.core.indexes import PandasIndex

store_path = tmp_path / "tmp.zarr"
original_ds = xr.Dataset({"data": ("x", np.arange(3))}, coords={"x": [-1, 0, 1]})
original_ds.to_zarr(store_path, mode="w")

with open_dataset(
store_path, engine="zarr", create_default_indexes=create_default_indexes
) as loaded_ds:
if create_default_indexes:
assert list(loaded_ds.xindexes) == ["x"] and isinstance(
loaded_ds.xindexes["x"], PandasIndex
)
else:
assert len(loaded_ds.xindexes) == 0


@requires_zarr
@pytest.mark.usefixtures("default_zarr_format")
def test_raises_key_error_on_invalid_zarr_store(tmp_path):
Expand Down
2 changes: 1 addition & 1 deletion xarray/tests/test_backends_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ def test_default_indexes(self, create_default_indexes):
if create_default_indexes:
assert all(name in final.xindexes for name in ["x", "y"])
else:
assert not final.xindexes
assert len(final.xindexes) == 0

@pytest.mark.parametrize("create_default_indexes", [True, False])
def test_default_indexes_passthrough(self, create_default_indexes):
Expand Down
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy