align tests with new time resolution behaviour

pydata · kmuehlbauer · Jan 15, 2025 · Oct 10, 2024 · Oct 10, 2024 · Oct 10, 2024
commit 8784f33a739c873088034020a666385f6949c84a
diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py
@@ -323,7 +323,7 @@ def create_test_data(
             f'Not enough letters for filling this dimension size ({_dims["dim3"]})'
         )
     obj["dim3"] = ("dim3", list(string.ascii_lowercase[0 : _dims["dim3"]]))
-    obj["time"] = ("time", pd.date_range("2000-01-01", periods=20))
+    obj["time"] = ("time", pd.date_range("2000-01-01", periods=20, unit="s"))
     for v, dims in sorted(_vars.items()):
         data = rs.normal(size=tuple(_dims[d] for d in dims))
         obj[v] = (dims, data)

diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
@@ -584,6 +584,12 @@ def test_roundtrip_cftime_datetime_data(self) -> None:
                     warnings.filterwarnings("ignore", "Unable to decode time axis")
 
                 with self.roundtrip(expected, save_kwargs=kwargs) as actual:
+                    # proleptic gregorian will be decoded into numpy datetime64
+                    # fixing to expectations
+                    if actual.t.dtype.kind == "M":
+                        dtype = f"datetime64[{np.datetime_data(actual.t)[0]}]"
+                        expected_decoded_t = expected_decoded_t.astype(dtype)
+                        expected_decoded_t0 = expected_decoded_t0.astype(dtype)
                     abs_diff = abs(actual.t.values - expected_decoded_t)
                     assert (abs_diff <= np.timedelta64(1, "s")).all()
                     assert (
@@ -598,7 +604,8 @@ def test_roundtrip_cftime_datetime_data(self) -> None:
                     assert actual.t.encoding["calendar"] == expected_calendar
 
     def test_roundtrip_timedelta_data(self) -> None:
-        time_deltas = pd.to_timedelta(["1h", "2h", "NaT"])  # type: ignore[arg-type, unused-ignore]
+        # todo: check, if default unit "s" is enough
+        time_deltas = pd.to_timedelta(["1h", "2h", "NaT"]).as_unit("s")  # type: ignore[arg-type, unused-ignore]
         expected = Dataset({"td": ("td", time_deltas), "td0": time_deltas[0]})
         with self.roundtrip(expected) as actual:
             assert_identical(expected, actual)
@@ -1583,7 +1590,8 @@ def test_open_encodings(self) -> None:
 
             expected = Dataset()
 
-            time = pd.date_range("1999-01-05", periods=10)
+            # todo: check, if specifying "s" is enough
+            time = pd.date_range("1999-01-05", periods=10, unit="s")
             encoding = {"units": units, "dtype": np.dtype("int32")}
             expected["time"] = ("time", time, {}, encoding)
 
@@ -5379,11 +5387,12 @@ def test_use_cftime_standard_calendar_default_in_range(calendar) -> None:
 
 @requires_cftime
 @requires_scipy_or_netCDF4
-@pytest.mark.parametrize("calendar", _STANDARD_CALENDARS)
-@pytest.mark.parametrize("units_year", [1500, 2500])
+@pytest.mark.parametrize("calendar", ["standard", "gregorian"])
+@pytest.mark.parametrize("units_year", [1500, 1582])
 def test_use_cftime_standard_calendar_default_out_of_range(
     calendar, units_year
 ) -> None:
+    # todo: check, if we still need to test for two dates
     import cftime
 
     x = [0, 1]
@@ -5472,9 +5481,10 @@ def test_use_cftime_false_standard_calendar_in_range(calendar) -> None:
 
 
 @requires_scipy_or_netCDF4
-@pytest.mark.parametrize("calendar", _STANDARD_CALENDARS)
-@pytest.mark.parametrize("units_year", [1500, 2500])
+@pytest.mark.parametrize("calendar", ["standard", "gregorian"])
+@pytest.mark.parametrize("units_year", [1500, 1582])
 def test_use_cftime_false_standard_calendar_out_of_range(calendar, units_year) -> None:
+    # todo: check, if we still need to check for two dates
     x = [0, 1]
     time = [0, 720]
     units = f"days since {units_year}-01-01"

diff --git a/xarray/tests/test_cftime_offsets.py b/xarray/tests/test_cftime_offsets.py
@@ -1496,7 +1496,7 @@ def test_date_range_like_same_calendar():
     assert src is out
 
 
-@pytest.mark.filterwarnings("ignore:Converting non-nanosecond")
+@pytest.mark.filterwarnings("ignore:Converting non-default")
 def test_date_range_like_errors():
     src = date_range("1899-02-03", periods=20, freq="D", use_cftime=False)
     src = src[np.arange(20) != 10]  # Remove 1 day so the frequency is not inferable.

diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py
@@ -1159,7 +1159,7 @@ def test_strftime_of_cftime_array(calendar):
 @pytest.mark.parametrize("unsafe", [False, True])
 def test_to_datetimeindex(calendar, unsafe):
     index = xr.cftime_range("2000", periods=5, calendar=calendar)
-    expected = pd.date_range("2000", periods=5)
+    expected = pd.date_range("2000", periods=5, unit="us")
 
     if calendar in _NON_STANDARD_CALENDARS and not unsafe:
         with pytest.warns(RuntimeWarning, match="non-standard"):
@@ -1176,7 +1176,11 @@ def test_to_datetimeindex(calendar, unsafe):
 @pytest.mark.parametrize("calendar", _ALL_CALENDARS)
 def test_to_datetimeindex_out_of_range(calendar):
     index = xr.cftime_range("0001", periods=5, calendar=calendar)
-    with pytest.raises(ValueError, match="0001"):
+    # todo: needs discussion, do we need this test?
+    if calendar in _NON_STANDARD_CALENDARS:
+        with pytest.warns(RuntimeWarning, match="non-standard"):
+            index.to_datetimeindex()
+    else:
         index.to_datetimeindex()
 
 
@@ -1199,7 +1203,8 @@ def test_multiindex():
 @pytest.mark.parametrize("freq", ["3663s", "33min", "2h"])
 @pytest.mark.parametrize("method", ["floor", "ceil", "round"])
 def test_rounding_methods_against_datetimeindex(freq, method):
-    expected = pd.date_range("2000-01-02T01:03:51", periods=10, freq="1777s")
+    # todo: check, if setting to "us" is enough
+    expected = pd.date_range("2000-01-02T01:03:51", periods=10, freq="1777s", unit="us")
     expected = getattr(expected, method)(freq)
     result = xr.cftime_range("2000-01-02T01:03:51", periods=10, freq="1777s")
     result = getattr(result, method)(freq).to_datetimeindex()

diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py
@@ -129,15 +129,20 @@ def test_cf_datetime(num_dates, units, calendar) -> None:
     expected = cftime.num2date(
         num_dates, units, calendar, only_use_cftime_datetimes=True
     )
-    min_y = np.ravel(np.atleast_1d(expected))[np.nanargmin(num_dates)].year
-    max_y = np.ravel(np.atleast_1d(expected))[np.nanargmax(num_dates)].year
-    if min_y >= 1678 and max_y < 2262:
+    print("0:", expected)
+
+    min_y = np.ravel(np.atleast_1d(expected))[np.nanargmin(num_dates)]  # .year
+    max_y = np.ravel(np.atleast_1d(expected))[np.nanargmax(num_dates)]  # .year
+    typ = type(min_y)
+    border = typ(1582, 10, 15)
+    if calendar == "proleptic_gregorian" or (min_y >= border and max_y >= border):
         expected = cftime_to_nptime(expected)
+    print("1:", expected)
 
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", "Unable to decode time axis")
         actual = decode_cf_datetime(num_dates, units, calendar)
-
+    print("2:", actual, type(actual), actual.dtype)
     abs_diff = np.asarray(abs(actual - expected)).ravel()
     abs_diff = pd.to_timedelta(abs_diff.tolist()).to_numpy()
 
@@ -146,6 +151,8 @@ def test_cf_datetime(num_dates, units, calendar) -> None:
     # https://github.com/Unidata/netcdf4-python/issues/355
     assert (abs_diff <= np.timedelta64(1, "s")).all()
     encoded1, _, _ = encode_cf_datetime(actual, units, calendar)
+
+    print("1:", encoded1)
     assert_duckarray_allclose(num_dates, encoded1)
 
     if hasattr(num_dates, "ndim") and num_dates.ndim == 1 and "1000" not in units:
@@ -211,14 +218,21 @@ def test_decode_standard_calendar_inside_timestamp_range(calendar) -> None:
     import cftime
 
     units = "days since 0001-01-01"
-    times = pd.date_range("2001-04-01-00", end="2001-04-30-23", freq="h")
+    unit = "us"
+    times = pd.date_range("2001-04-01-00", end="2001-04-30-23", unit=unit, freq="h")
+    print("A:", times)
     time = cftime.date2num(times.to_pydatetime(), units, calendar=calendar)
     expected = times.values
-    expected_dtype = np.dtype("M8[ns]")
-
+    if calendar == "proleptic_gregorian":
+        unit = "s"
+    expected_dtype = np.dtype(f"M8[{unit}]")
+    print("0:", time[0], units, calendar)
+    print("1:", expected.astype("int64")[0], expected.dtype)
     actual = decode_cf_datetime(time, units, calendar=calendar)
+    print("2:", actual[0], actual.astype("int64")[0], actual.dtype)
     assert actual.dtype == expected_dtype
     abs_diff = abs(actual - expected)
+    print("3:", abs_diff[0], abs_diff.dtype)
     # once we no longer support versions of netCDF4 older than 1.1.5,
     # we could do this check with near microsecond accuracy:
     # https://github.com/Unidata/netcdf4-python/issues/355
@@ -262,6 +276,9 @@ def test_decode_dates_outside_timestamp_range(calendar) -> None:
     expected = cftime.num2date(
         time, units, calendar=calendar, only_use_cftime_datetimes=True
     )
+    # special case proleptic_gregorian
+    if calendar == "proleptic_gregorian":
+        expected = expected.astype("=M8[us]")
     expected_date_type = type(expected[0])
 
     with warnings.catch_warnings():
@@ -272,7 +289,7 @@ def test_decode_dates_outside_timestamp_range(calendar) -> None:
     # once we no longer support versions of netCDF4 older than 1.1.5,
     # we could do this check with near microsecond accuracy:
     # https://github.com/Unidata/netcdf4-python/issues/355
-    assert (abs_diff <= np.timedelta64(1, "s")).all()
+    assert (abs_diff <= np.timedelta64(1, "us")).all()
 
 
 @requires_cftime
@@ -281,11 +298,13 @@ def test_decode_standard_calendar_single_element_inside_timestamp_range(
     calendar,
 ) -> None:
     units = "days since 0001-01-01"
+    unit = "s" if calendar == "proleptic_gregorian" else "us"
     for num_time in [735368, [735368], [[735368]]]:
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", "Unable to decode time axis")
             actual = decode_cf_datetime(num_time, units, calendar=calendar)
-        assert actual.dtype == np.dtype("M8[ns]")
+
+        assert actual.dtype == np.dtype(f"M8[{unit}]")
 
 
 @requires_cftime
@@ -327,6 +346,7 @@ def test_decode_standard_calendar_multidim_time_inside_timestamp_range(
     import cftime
 
     units = "days since 0001-01-01"
+    unit = "s" if calendar == "proleptic_gregorian" else "us"
     times1 = pd.date_range("2001-04-01", end="2001-04-05", freq="D")
     times2 = pd.date_range("2001-05-01", end="2001-05-05", freq="D")
     time1 = cftime.date2num(times1.to_pydatetime(), units, calendar=calendar)
@@ -339,7 +359,7 @@ def test_decode_standard_calendar_multidim_time_inside_timestamp_range(
     expected2 = times2.values
 
     actual = decode_cf_datetime(mdim_time, units, calendar=calendar)
-    assert actual.dtype == np.dtype("M8[ns]")
+    assert actual.dtype == np.dtype(f"M8[{unit}]")
 
     abs_diff1 = abs(actual[:, 0] - expected1)
     abs_diff2 = abs(actual[:, 1] - expected2)
@@ -414,7 +434,14 @@ def test_decode_multidim_time_outside_timestamp_range(calendar) -> None:
         warnings.filterwarnings("ignore", "Unable to decode time axis")
         actual = decode_cf_datetime(mdim_time, units, calendar=calendar)
 
-    assert actual.dtype == np.dtype("O")
+    if calendar == "proleptic_gregorian":
+        dtype = np.dtype("=M8[s]")
+        expected1 = expected1.astype(dtype)
+        expected2 = expected2.astype(dtype)
+    else:
+        dtype = np.dtype("O")
+
+    assert actual.dtype == dtype
 
     abs_diff1 = abs(actual[:, 0] - expected1)
     abs_diff2 = abs(actual[:, 1] - expected2)
@@ -509,7 +536,7 @@ def test_decoded_cf_datetime_array_2d() -> None:
         ("x", "y"), np.array([[0, 1], [2, 3]]), {"units": "days since 2000-01-01"}
     )
     result = CFDatetimeCoder().decode(variable)
-    assert result.dtype == "datetime64[ns]"
+    assert result.dtype == "datetime64[s]"
     expected = pd.date_range("2000-01-01", periods=4).values.reshape(2, 2)
     assert_array_equal(np.asarray(result), expected)
 
@@ -675,7 +702,7 @@ def test_decode_cf(calendar) -> None:
         if calendar not in _STANDARD_CALENDARS:
             assert ds.test.dtype == np.dtype("O")
         else:
-            assert ds.test.dtype == np.dtype("M8[ns]")
+            assert ds.test.dtype == np.dtype("M8[s]")
 
 
 def test_decode_cf_time_bounds() -> None:
@@ -700,7 +727,7 @@ def test_decode_cf_time_bounds() -> None:
         "calendar": "standard",
     }
     dsc = decode_cf(ds)
-    assert dsc.time_bnds.dtype == np.dtype("M8[ns]")
+    assert dsc.time_bnds.dtype == np.dtype("M8[s]")
     dsc = decode_cf(ds, decode_times=False)
     assert dsc.time_bnds.dtype == np.dtype("int64")
 
@@ -917,8 +944,8 @@ def test_use_cftime_default_standard_calendar_in_range(calendar) -> None:
 
 
 @requires_cftime
-@pytest.mark.parametrize("calendar", _STANDARD_CALENDARS)
-@pytest.mark.parametrize("units_year", [1500, 2500])
+@pytest.mark.parametrize("calendar", ["standard", "gregorian"])
+@pytest.mark.parametrize("units_year", [1500, 1580])
 def test_use_cftime_default_standard_calendar_out_of_range(
     calendar, units_year
 ) -> None:
@@ -980,8 +1007,8 @@ def test_use_cftime_false_standard_calendar_in_range(calendar) -> None:
         np.testing.assert_array_equal(result, expected)
 
 
-@pytest.mark.parametrize("calendar", _STANDARD_CALENDARS)
-@pytest.mark.parametrize("units_year", [1500, 2500])
+@pytest.mark.parametrize("calendar", ["standard", "gregorian"])
+@pytest.mark.parametrize("units_year", [1500, 1582])
 def test_use_cftime_false_standard_calendar_out_of_range(calendar, units_year) -> None:
     numerical_dates = [0, 1]
     units = f"days since {units_year}-01-01"
@@ -1266,7 +1293,7 @@ def test_roundtrip_datetime64_nanosecond_precision(
         encoding = {}
 
     var = Variable(["time"], times, encoding=encoding)
-    assert var.dtype == np.dtype("=M8[ns]")
+    assert var.dtype == np.dtype(f"=M8[{timeunit}]")
 
     encoded_var = conventions.encode_cf_variable(var)
     assert (
@@ -1277,7 +1304,7 @@ def test_roundtrip_datetime64_nanosecond_precision(
     assert encoded_var.data.dtype == dtype
 
     decoded_var = conventions.decode_cf_variable("foo", encoded_var)
-    assert decoded_var.dtype == np.dtype("=M8[ns]")
+    assert decoded_var.dtype == np.dtype(f"=M8[{timeunit}]")
     assert (
         decoded_var.encoding["units"]
         == f"{_numpy_to_netcdf_timeunit(timeunit)} since 1970-01-01 00:00:00"

diff --git a/xarray/tests/test_concat.py b/xarray/tests/test_concat.py
@@ -319,7 +319,7 @@ def test_concat_multiple_datasets_with_multiple_missing_variables() -> None:
     assert_identical(actual, expected)
 
 
-@pytest.mark.filterwarnings("ignore:Converting non-nanosecond")
+@pytest.mark.filterwarnings("ignore:Converting non-default")
 def test_concat_type_of_missing_fill() -> None:
     datasets = create_typed_datasets(2, seed=123)
     expected1 = concat(datasets, dim="day", fill_value=dtypes.NA)

diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py
@@ -213,7 +213,7 @@ def test_deterministic_coords_encoding(self) -> None:
         vars, attrs = conventions.encode_dataset_coordinates(ds)
         assert attrs["coordinates"] == "bar baz"
 
-    @pytest.mark.filterwarnings("ignore:Converting non-nanosecond")
+    @pytest.mark.filterwarnings("ignore:Converting non-default")
     def test_emit_coordinates_attribute_in_attrs(self) -> None:
         orig = Dataset(
             {"a": 1, "b": 1},
@@ -231,7 +231,7 @@ def test_emit_coordinates_attribute_in_attrs(self) -> None:
         assert enc["b"].attrs.get("coordinates") == "t"
         assert "coordinates" not in enc["b"].encoding
 
-    @pytest.mark.filterwarnings("ignore:Converting non-nanosecond")
+    @pytest.mark.filterwarnings("ignore:Converting non-default")
     def test_emit_coordinates_attribute_in_encoding(self) -> None:
         orig = Dataset(
             {"a": 1, "b": 1},
@@ -364,7 +364,7 @@ def test_dataset_repr_with_netcdf4_datetimes(self) -> None:
 
         attrs = {"units": "days since 1900-01-01"}
         ds = decode_cf(Dataset({"time": ("time", [0, 1], attrs)}))
-        assert "(time) datetime64[ns]" in repr(ds)
+        assert "(time) datetime64[s]" in repr(ds)
 
     @requires_cftime
     def test_decode_cf_datetime_transition_to_invalid(self) -> None:
@@ -447,13 +447,13 @@ def test_decode_cf_time_kwargs(self) -> None:
 
         dsc = conventions.decode_cf(ds)
         assert dsc.timedelta.dtype == np.dtype("m8[ns]")
-        assert dsc.time.dtype == np.dtype("M8[ns]")
+        assert dsc.time.dtype == np.dtype("M8[s]")
         dsc = conventions.decode_cf(ds, decode_times=False)
         assert dsc.timedelta.dtype == np.dtype("int64")
         assert dsc.time.dtype == np.dtype("int64")
         dsc = conventions.decode_cf(ds, decode_times=True, decode_timedelta=False)
         assert dsc.timedelta.dtype == np.dtype("int64")
-        assert dsc.time.dtype == np.dtype("M8[ns]")
+        assert dsc.time.dtype == np.dtype("M8[s]")
         dsc = conventions.decode_cf(ds, decode_times=False, decode_timedelta=True)
         assert dsc.timedelta.dtype == np.dtype("m8[ns]")
         assert dsc.time.dtype == np.dtype("int64")

diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py
@@ -3657,7 +3657,7 @@ def test_to_and_from_dict(
         actual_no_data = da.to_dict(data=False, encoding=encoding)
         assert expected_no_data == actual_no_data
 
-    @pytest.mark.filterwarnings("ignore:Converting non-nanosecond")
+    @pytest.mark.filterwarnings("ignore:Converting non-default")
     def test_to_and_from_dict_with_time_dim(self) -> None:
         x = np.random.randn(10, 3)
         t = pd.date_range("20130101", periods=10)
@@ -3666,7 +3666,7 @@ def test_to_and_from_dict_with_time_dim(self) -> None:
         roundtripped = DataArray.from_dict(da.to_dict())
         assert_identical(da, roundtripped)
 
-    @pytest.mark.filterwarnings("ignore:Converting non-nanosecond")
+    @pytest.mark.filterwarnings("ignore:Converting non-default")
     def test_to_and_from_dict_with_nan_nat(self) -> None:
         y = np.random.randn(10, 3)
         y[2] = np.nan