From 54475ed74996151f7c6b956387e884b4d1521256 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Thu, 6 Feb 2025 14:16:40 -0500 Subject: [PATCH 01/12] Upgrade mypy to 1.15 Mypy 1.15 includes fix for , allowing several "type: ignore" comments to be removed. --- .github/workflows/ci-additional.yaml | 4 +-- .pre-commit-config.yaml | 38 +++++++++++++++++++--------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml index fef104a1ddc..0619fa1a552 100644 --- a/.github/workflows/ci-additional.yaml +++ b/.github/workflows/ci-additional.yaml @@ -125,7 +125,7 @@ jobs: - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@v5.3.1 with: - file: mypy_report/cobertura.xml + files: mypy_report/cobertura.xml flags: mypy env_vars: PYTHON_VERSION name: codecov-umbrella @@ -176,7 +176,7 @@ jobs: - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@v5.3.1 with: - file: mypy_report/cobertura.xml + files: mypy_report/cobertura.xml flags: mypy-min env_vars: PYTHON_VERSION name: codecov-umbrella diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f02dbf9dc69..3ee39291c3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,23 +42,37 @@ repos: - id: prettier args: [--cache-location=.prettier_cache/cache] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.14.1 + rev: v1.15.0 hooks: - id: mypy - # Copied from setup.cfg - exclude: "properties|asv_bench" + files: "^xarray" + exclude: "^xarray/util/generate_.*\\.py$" # This is slow and so we take it out of the fast-path; requires passing # `--hook-stage manual` to pre-commit stages: [manual] - additional_dependencies: [ - # Type stubs - types-python-dateutil, - types-setuptools, - types-PyYAML, - types-pytz, - typing-extensions>=4.1.0, - numpy, - ] + additional_dependencies: + # Type stubs plus additional dependencies from ci/requirements/environment.yml + # required in order to satisfy most (ideally all) type checks. This is rather + # brittle, so it is difficult (if not impossible) to get mypy to succeed in + # this context, even when it succeeds in CI. + - dask + - distributed + - hypothesis + - matplotlib + - numpy==2.1.3 + - pandas-stubs + - pytest + - types-colorama + - types-defusedxml + - types-docutils + - types-pexpect + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools + - typing-extensions>=4.1.0 - repo: https://github.com/citation-file-format/cff-converter-python rev: ebf0b5e44d67f8beaa1cd13a0d0393ea04c6058d hooks: From 369bb327cae26d6ee9b94a6fa1e0709ac86b13cc Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Fri, 7 Feb 2025 15:45:14 -0500 Subject: [PATCH 02/12] Add type annotations to DataTree.pipe tests --- xarray/tests/test_datatree.py | 40 +++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/xarray/tests/test_datatree.py b/xarray/tests/test_datatree.py index 715d80e084a..c87a1e1329e 100644 --- a/xarray/tests/test_datatree.py +++ b/xarray/tests/test_datatree.py @@ -1,7 +1,7 @@ import re import sys import typing -from collections.abc import Mapping +from collections.abc import Callable, Mapping from copy import copy, deepcopy from textwrap import dedent @@ -1589,27 +1589,53 @@ def test_assign(self) -> None: class TestPipe: - def test_noop(self, create_test_datatree) -> None: + def test_noop(self, create_test_datatree: Callable[[], DataTree]) -> None: dt = create_test_datatree() actual = dt.pipe(lambda tree: tree) assert actual.identical(dt) - def test_params(self, create_test_datatree) -> None: + def test_args(self, create_test_datatree: Callable[[], DataTree]) -> None: dt = create_test_datatree() - def f(tree, **attrs): - return tree.assign(arr_with_attrs=xr.Variable("dim0", [], attrs=attrs)) + def f(tree: DataTree, x: int, y: int) -> DataTree: + return tree.assign( + arr_with_attrs=xr.Variable("dim0", [], attrs=dict(x=x, y=y)) + ) + + actual = dt.pipe(f, 1, 2) + assert actual["arr_with_attrs"].attrs == dict(x=1, y=2) + + def test_kwargs(self, create_test_datatree: Callable[[], DataTree]) -> None: + dt = create_test_datatree() + + def f(tree: DataTree, *, x: int, y: int, z: int) -> DataTree: + return tree.assign( + arr_with_attrs=xr.Variable("dim0", [], attrs=dict(x=x, y=y, z=z)) + ) attrs = {"x": 1, "y": 2, "z": 3} actual = dt.pipe(f, **attrs) assert actual["arr_with_attrs"].attrs == attrs - def test_named_self(self, create_test_datatree) -> None: + def test_args_kwargs(self, create_test_datatree: Callable[[], DataTree]) -> None: + dt = create_test_datatree() + + def f(tree: DataTree, x: int, *, y: int, z: int) -> DataTree: + return tree.assign( + arr_with_attrs=xr.Variable("dim0", [], attrs=dict(x=x, y=y, z=z)) + ) + + attrs = {"x": 1, "y": 2, "z": 3} + + actual = dt.pipe(f, attrs["x"], y=attrs["y"], z=attrs["z"]) + assert actual["arr_with_attrs"].attrs == attrs + + def test_named_self(self, create_test_datatree: Callable[[], DataTree]) -> None: dt = create_test_datatree() - def f(x, tree, y): + def f(x: int, tree: DataTree, y: int): tree.attrs.update({"x": x, "y": y}) return tree From 9815d61ce0d99487431139935c47962591ecbe0a Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Fri, 7 Feb 2025 16:46:29 -0500 Subject: [PATCH 03/12] More precisely type `pipe` methods. In addition, enhance mypy job configuration to support running it locally via `act`. Fixes #9997 --- .github/workflows/ci-additional.yaml | 80 ++++------- xarray/core/common.py | 33 ++++- xarray/core/datatree.py | 68 +++++++-- xarray/tests/test_dataarray_typing.yml | 190 +++++++++++++++++++++++++ xarray/tests/test_dataset_typing.yml | 190 +++++++++++++++++++++++++ xarray/tests/test_datatree_typing.yml | 190 +++++++++++++++++++++++++ 6 files changed, 676 insertions(+), 75 deletions(-) create mode 100644 xarray/tests/test_dataarray_typing.yml create mode 100644 xarray/tests/test_dataset_typing.yml create mode 100644 xarray/tests/test_datatree_typing.yml diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml index 0619fa1a552..b0948dd2e45 100644 --- a/.github/workflows/ci-additional.yaml +++ b/.github/workflows/ci-additional.yaml @@ -33,6 +33,15 @@ jobs: with: keyword: "[skip-ci]" + detect-act: + name: Detect 'act' runner + runs-on: ubuntu-latest + outputs: + running: ${{ steps.detect-act.outputs.running }} + steps: + - id: detect-act + run: echo "running=${{ env.ACT || 'false' }}" >> "$GITHUB_OUTPUT" + doctest: name: Doctests runs-on: "ubuntu-latest" @@ -81,15 +90,23 @@ jobs: python -m pytest --doctest-modules xarray --ignore xarray/tests -Werror mypy: - name: Mypy + strategy: + matrix: + include: + - python-version: "3.10" + codecov-flags: mypy-min + - python-version: "3.12" + codecov-flags: mypy + name: Mypy ${{ matrix.python-version }} runs-on: "ubuntu-latest" - needs: detect-ci-trigger + needs: [detect-ci-trigger, detect-act] + if: always() && (needs.detect-ci-trigger.outputs.triggered == 'true' || needs.detect-act.outputs.running == 'true') defaults: run: shell: bash -l {0} env: CONDA_ENV_FILE: ci/requirements/environment.yml - PYTHON_VERSION: "3.12" + PYTHON_VERSION: ${{ matrix.python-version }} steps: - uses: actions/checkout@v4 @@ -116,68 +133,23 @@ jobs: python xarray/util/print_versions.py - name: Install mypy run: | - python -m pip install "mypy==1.15" --force-reinstall + python -m pip install "mypy==1.15" pytest-mypy-plugins --force-reinstall - name: Run mypy run: | python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report - - name: Upload mypy coverage to Codecov - uses: codecov/codecov-action@v5.3.1 - with: - files: mypy_report/cobertura.xml - flags: mypy - env_vars: PYTHON_VERSION - name: codecov-umbrella - fail_ci_if_error: false - - mypy-min: - name: Mypy 3.10 - runs-on: "ubuntu-latest" - needs: detect-ci-trigger - defaults: - run: - shell: bash -l {0} - env: - CONDA_ENV_FILE: ci/requirements/environment.yml - PYTHON_VERSION: "3.10" - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all branches and tags. - - - name: set environment variables + - name: Run mypy tests + # Run pytest with mypy plugin even if mypy analysis in previous step fails. + if: ${{ always() }} run: | - echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v2 - with: - environment-file: ${{env.CONDA_ENV_FILE}} - environment-name: xarray-tests - create-args: >- - python=${{env.PYTHON_VERSION}} - cache-environment: true - cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" - - name: Install xarray - run: | - python -m pip install --no-deps -e . - - name: Version info - run: | - python xarray/util/print_versions.py - - name: Install mypy - run: | - python -m pip install "mypy==1.15" --force-reinstall - - - name: Run mypy - run: | - python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report + python -m pytest -v --mypy-only-local-stub --mypy-pyproject-toml-file=pyproject.toml xarray/**/test_*.yml - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@v5.3.1 with: files: mypy_report/cobertura.xml - flags: mypy-min + flags: ${{ matrix.codecov-flags }} env_vars: PYTHON_VERSION name: codecov-umbrella fail_ci_if_error: false diff --git a/xarray/core/common.py b/xarray/core/common.py index 01c02a8d14f..ceaae42356a 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -6,7 +6,7 @@ from contextlib import suppress from html import escape from textwrap import dedent -from typing import TYPE_CHECKING, Any, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, Union, overload import numpy as np import pandas as pd @@ -60,6 +60,7 @@ T_Resample = TypeVar("T_Resample", bound="Resample") C = TypeVar("C") T = TypeVar("T") +P = ParamSpec("P") class ImplementsArrayReduce: @@ -718,11 +719,27 @@ def assign_attrs(self, *args: Any, **kwargs: Any) -> Self: out.attrs.update(*args, **kwargs) return out + @overload + def pipe( + self, + func: Callable[Concatenate[Self, P], T], + *args: P.args, + **kwargs: P.kwargs, + ) -> T: ... + + @overload def pipe( self, - func: Callable[..., T] | tuple[Callable[..., T], str], + func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any, + ) -> T: ... + + def pipe( + self, + func: Callable[Concatenate[Self, P], T] | tuple[Callable[P, T], str], + *args: P.args, + **kwargs: P.kwargs, ) -> T: """ Apply ``func(self, *args, **kwargs)`` @@ -840,15 +857,19 @@ def pipe( pandas.DataFrame.pipe """ if isinstance(func, tuple): - func, target = func + # Use different var when unpacking function from tuple because the type + # signature of the unpacked function differs from the expected type + # signature in the case where only a function is given, rather than a tuple. + # This makes type checkers happy at both call sites below. + f, target = func if target in kwargs: raise ValueError( f"{target} is both the pipe target and a keyword argument" ) kwargs[target] = self - return func(*args, **kwargs) - else: - return func(self, *args, **kwargs) + return f(*args, **kwargs) + + return func(self, *args, **kwargs) def rolling_exp( self: T_DataWithCoords, diff --git a/xarray/core/datatree.py b/xarray/core/datatree.py index 1a388919f0c..61340ac99ad 100644 --- a/xarray/core/datatree.py +++ b/xarray/core/datatree.py @@ -12,7 +12,17 @@ Mapping, ) from html import escape -from typing import TYPE_CHECKING, Any, Literal, NoReturn, Union, overload +from typing import ( + TYPE_CHECKING, + Any, + Concatenate, + Literal, + NoReturn, + ParamSpec, + TypeVar, + Union, + overload, +) from xarray.core import utils from xarray.core._aggregations import DataTreeAggregations @@ -80,18 +90,23 @@ # """ # DEVELOPERS' NOTE # ---------------- -# The idea of this module is to create a `DataTree` class which inherits the tree structure from TreeNode, and also copies -# the entire API of `xarray.Dataset`, but with certain methods decorated to instead map the dataset function over every -# node in the tree. As this API is copied without directly subclassing `xarray.Dataset` we instead create various Mixin -# classes (in ops.py) which each define part of `xarray.Dataset`'s extensive API. +# The idea of this module is to create a `DataTree` class which inherits the tree +# structure from TreeNode, and also copies the entire API of `xarray.Dataset`, but with +# certain methods decorated to instead map the dataset function over every node in the +# tree. As this API is copied without directly subclassing `xarray.Dataset` we instead +# create various Mixin classes (in ops.py) which each define part of `xarray.Dataset`'s +# extensive API. # -# Some of these methods must be wrapped to map over all nodes in the subtree. Others are fine to inherit unaltered -# (normally because they (a) only call dataset properties and (b) don't return a dataset that should be nested into a new -# tree) and some will get overridden by the class definition of DataTree. +# Some of these methods must be wrapped to map over all nodes in the subtree. Others are +# fine to inherit unaltered (normally because they (a) only call dataset properties and +# (b) don't return a dataset that should be nested into a new tree) and some will get +# overridden by the class definition of DataTree. # """ T_Path = Union[str, NodePath] +T = TypeVar("T") +P = ParamSpec("P") def _collect_data_and_coord_variables( @@ -1465,9 +1480,28 @@ def map_over_datasets( # TODO fix this typing error return map_over_datasets(func, self, *args, kwargs=kwargs) + @overload + def pipe( + self, + func: Callable[Concatenate[Self, P], T], + *args: P.args, + **kwargs: P.kwargs, + ) -> T: ... + + @overload + def pipe( + self, + func: tuple[Callable[..., T], str], + *args: Any, + **kwargs: Any, + ) -> T: ... + def pipe( - self, func: Callable | tuple[Callable, str], *args: Any, **kwargs: Any - ) -> Any: + self, + func: Callable[Concatenate[Self, P], T] | tuple[Callable[..., T], str], + *args: Any, + **kwargs: Any, + ) -> T: """Apply ``func(self, *args, **kwargs)`` This method replicates the pandas method of the same name. @@ -1487,7 +1521,7 @@ def pipe( Returns ------- - object : Any + object : T the return type of ``func``. Notes @@ -1515,15 +1549,19 @@ def pipe( """ if isinstance(func, tuple): - func, target = func + # Use different var when unpacking function from tuple because the type + # signature of the unpacked function differs from the expected type + # signature in the case where only a function is given, rather than a tuple. + # This makes type checkers happy at both call sites below. + f, target = func if target in kwargs: raise ValueError( f"{target} is both the pipe target and a keyword argument" ) kwargs[target] = self - else: - args = (self,) + args - return func(*args, **kwargs) + return f(*args, **kwargs) + + return func(self, *args, **kwargs) # TODO some kind of .collapse() or .flatten() method to merge a subtree diff --git a/xarray/tests/test_dataarray_typing.yml b/xarray/tests/test_dataarray_typing.yml new file mode 100644 index 00000000000..04a6ee452aa --- /dev/null +++ b/xarray/tests/test_dataarray_typing.yml @@ -0,0 +1,190 @@ +- case: test_pipe_lambda_noarg_return_type + main: | + from xarray import DataArray + + da = DataArray().pipe(lambda data: data) + + reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" + +- case: test_pipe_lambda_posarg_return_type + main: | + from xarray import DataArray + + da = DataArray().pipe(lambda data, arg: arg, "foo") + + reveal_type(da) # N: Revealed type is "builtins.str" + +- case: test_pipe_lambda_chaining_return_type + main: | + from xarray import DataArray + + answer = DataArray().pipe(lambda data, arg: arg, "foo").count("o") + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_lambda_missing_arg + main: | + from xarray import DataArray + + # Call to pipe missing argument for lambda parameter `arg` + da = DataArray().pipe(lambda data, arg: data) + out: | + main:4: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Any, Any], Any]" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_lambda_extra_arg + main: | + from xarray import DataArray + + # Call to pipe with extra argument for lambda + da = DataArray().pipe(lambda data: data, "oops!") + out: | + main:4: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Any], Any]", "str" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_posarg + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int) -> DataArray: + return da + + # Call to pipe missing argument for function parameter `arg` + da = DataArray().pipe(f) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[DataArray, int], DataArray]" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_extra_posarg + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int) -> DataArray: + return da + + # Call to pipe missing keyword for kwonly parameter `kwonly` + da = DataArray().pipe(f, 42, "oops!") + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[DataArray, int], DataArray]", "int", "str" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_kwarg + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int, *, kwonly: int) -> DataArray: + return da + + # Call to pipe missing argument for kwonly parameter `kwonly` + da = DataArray().pipe(f, 42) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[DataArray, int, NamedArg(int, 'kwonly')], DataArray]", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_keyword + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int, *, kwonly: int) -> DataArray: + return da + + # Call to pipe missing keyword for kwonly parameter `kwonly` + da = DataArray().pipe(f, 42, 99) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[DataArray, int, NamedArg(int, 'kwonly')], DataArray]", "int", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_unexpected_keyword + main: | + from xarray import DataArray + + def f(da: DataArray, arg: int, *, kwonly: int) -> DataArray: + return da + + # Call to pipe using wrong keyword: `kw` instead of `kwonly` + da = DataArray().pipe(f, 42, kw=99) + out: | + main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataWithCoords" [call-arg] + +- case: test_pipe_tuple_return_type_dataarray + main: | + from xarray import DataArray + + def f(arg: int, da: DataArray) -> DataArray: + return da + + da = DataArray().pipe((f, "da"), 42) + reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" + +- case: test_pipe_tuple_return_type_other + main: | + from xarray import DataArray + + def f(arg: int, da: DataArray) -> int: + return arg + + answer = DataArray().pipe((f, "da"), 42) + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_tuple_missing_arg + main: | + from xarray import DataArray + + def f(arg: int, da: DataArray) -> DataArray: + return da + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are missing an argument for parameter `arg`, so we get no error here. + + da = DataArray().pipe((f, "da")) + reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we failed to pass an argument for `arg`. + + da = DataArray().pipe(lambda data, arg: f(arg, data)) + out: | + main:17: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Any, Any], DataArray]" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_tuple_extra_arg + main: | + from xarray import DataArray + + def f(arg: int, da: DataArray) -> DataArray: + return da + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are providing too many args for `f`, so we get no error here. + + da = DataArray().pipe((f, "da"), 42, "foo") + reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we passed too many arguments. + + da = DataArray().pipe(lambda data, arg: f(arg, data), 42, "foo") + out: | + main:17: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Any, Any], DataArray]", "int", "str" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T diff --git a/xarray/tests/test_dataset_typing.yml b/xarray/tests/test_dataset_typing.yml new file mode 100644 index 00000000000..99ea4554e84 --- /dev/null +++ b/xarray/tests/test_dataset_typing.yml @@ -0,0 +1,190 @@ +- case: test_pipe_lambda_noarg_return_type + main: | + from xarray import Dataset + + ds = Dataset().pipe(lambda data: data) + + reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" + +- case: test_pipe_lambda_posarg_return_type + main: | + from xarray import Dataset + + ds = Dataset().pipe(lambda data, arg: arg, "foo") + + reveal_type(ds) # N: Revealed type is "builtins.str" + +- case: test_pipe_lambda_chaining_return_type + main: | + from xarray import Dataset + + answer = Dataset().pipe(lambda data, arg: arg, "foo").count("o") + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_lambda_missing_arg + main: | + from xarray import Dataset + + # Call to pipe missing argument for lambda parameter `arg` + ds = Dataset().pipe(lambda data, arg: data) + out: | + main:4: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Any, Any], Any]" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_lambda_extra_arg + main: | + from xarray import Dataset + + # Call to pipe with extra argument for lambda + ds = Dataset().pipe(lambda data: data, "oops!") + out: | + main:4: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Any], Any]", "str" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_posarg + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int) -> Dataset: + return ds + + # Call to pipe missing argument for function parameter `arg` + ds = Dataset().pipe(f) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Dataset, int], Dataset]" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_extra_posarg + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int) -> Dataset: + return ds + + # Call to pipe missing keyword for kwonly parameter `kwonly` + ds = Dataset().pipe(f, 42, "oops!") + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Dataset, int], Dataset]", "int", "str" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_kwarg + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int, *, kwonly: int) -> Dataset: + return ds + + # Call to pipe missing argument for kwonly parameter `kwonly` + ds = Dataset().pipe(f, 42) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Dataset, int, NamedArg(int, 'kwonly')], Dataset]", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_keyword + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int, *, kwonly: int) -> Dataset: + return ds + + # Call to pipe missing keyword for kwonly parameter `kwonly` + ds = Dataset().pipe(f, 42, 99) + out: | + main:7: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Dataset, int, NamedArg(int, 'kwonly')], Dataset]", "int", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_unexpected_keyword + main: | + from xarray import Dataset + + def f(ds: Dataset, arg: int, *, kwonly: int) -> Dataset: + return ds + + # Call to pipe using wrong keyword: `kw` instead of `kwonly` + ds = Dataset().pipe(f, 42, kw=99) + out: | + main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataWithCoords" [call-arg] + +- case: test_pipe_tuple_return_type_dataset + main: | + from xarray import Dataset + + def f(arg: int, ds: Dataset) -> Dataset: + return ds + + ds = Dataset().pipe((f, "ds"), 42) + reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" + +- case: test_pipe_tuple_return_type_other + main: | + from xarray import Dataset + + def f(arg: int, ds: Dataset) -> int: + return arg + + answer = Dataset().pipe((f, "ds"), 42) + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_tuple_missing_arg + main: | + from xarray import Dataset + + def f(arg: int, ds: Dataset) -> Dataset: + return ds + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are missing an argument for parameter `arg`, so we get no error here. + + ds = Dataset().pipe((f, "ds")) + reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we failed to pass an argument for `arg`. + + ds = Dataset().pipe(lambda data, arg: f(arg, data)) + out: | + main:17: error: No overload variant of "pipe" of "DataWithCoords" matches argument type "Callable[[Any, Any], Dataset]" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_tuple_extra_arg + main: | + from xarray import Dataset + + def f(arg: int, ds: Dataset) -> Dataset: + return ds + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are providing too many args for `f`, so we get no error here. + + ds = Dataset().pipe((f, "ds"), 42, "foo") + reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we passed too many arguments. + + ds = Dataset().pipe(lambda data, arg: f(arg, data), 42, "foo") + out: | + main:17: error: No overload variant of "pipe" of "DataWithCoords" matches argument types "Callable[[Any, Any], Dataset]", "int", "str" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T diff --git a/xarray/tests/test_datatree_typing.yml b/xarray/tests/test_datatree_typing.yml new file mode 100644 index 00000000000..69d566d07e5 --- /dev/null +++ b/xarray/tests/test_datatree_typing.yml @@ -0,0 +1,190 @@ +- case: test_pipe_lambda_noarg_return_type + main: | + from xarray import DataTree + + dt = DataTree().pipe(lambda data: data) + + reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" + +- case: test_pipe_lambda_posarg_return_type + main: | + from xarray import DataTree + + dt = DataTree().pipe(lambda data, arg: arg, "foo") + + reveal_type(dt) # N: Revealed type is "builtins.str" + +- case: test_pipe_lambda_chaining_return_type + main: | + from xarray import DataTree + + answer = DataTree().pipe(lambda data, arg: arg, "foo").count("o") + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_lambda_missing_arg + main: | + from xarray import DataTree + + # Call to pipe missing argument for lambda parameter `arg` + dt = DataTree().pipe(lambda data, arg: data) + out: | + main:4: error: No overload variant of "pipe" of "DataTree" matches argument type "Callable[[Any, Any], Any]" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_lambda_extra_arg + main: | + from xarray import DataTree + + # Call to pipe with extra argument for lambda + dt = DataTree().pipe(lambda data: data, "oops!") + out: | + main:4: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[Any], Any]", "str" [call-overload] + main:4: note: Possible overload variants: + main:4: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_posarg + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int) -> DataTree: + return dt + + # Call to pipe missing argument for function parameter `arg` + dt = DataTree().pipe(f) + out: | + main:7: error: No overload variant of "pipe" of "DataTree" matches argument type "Callable[[DataTree, int], DataTree]" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_extra_posarg + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int) -> DataTree: + return dt + + # Call to pipe missing keyword for kwonly parameter `kwonly` + dt = DataTree().pipe(f, 42, "oops!") + out: | + main:7: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[DataTree, int], DataTree]", "int", "str" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_kwarg + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int, *, kwonly: int) -> DataTree: + return dt + + # Call to pipe missing argument for kwonly parameter `kwonly` + dt = DataTree().pipe(f, 42) + out: | + main:7: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[DataTree, int, NamedArg(int, 'kwonly')], DataTree]", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_missing_keyword + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int, *, kwonly: int) -> DataTree: + return dt + + # Call to pipe missing keyword for kwonly parameter `kwonly` + dt = DataTree().pipe(f, 42, 99) + out: | + main:7: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[DataTree, int, NamedArg(int, 'kwonly')], DataTree]", "int", "int" [call-overload] + main:7: note: Possible overload variants: + main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_function_unexpected_keyword + main: | + from xarray import DataTree + + def f(dt: DataTree, arg: int, *, kwonly: int) -> DataTree: + return dt + + # Call to pipe using wrong keyword: `kw` instead of `kwonly` + dt = DataTree().pipe(f, 42, kw=99) + out: | + main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataTree" [call-arg] + +- case: test_pipe_tuple_return_type_datatree + main: | + from xarray import DataTree + + def f(arg: int, dt: DataTree) -> DataTree: + return dt + + dt = DataTree().pipe((f, "dt"), 42) + reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" + +- case: test_pipe_tuple_return_type_other + main: | + from xarray import DataTree + + def f(arg: int, dt: DataTree) -> int: + return arg + + answer = DataTree().pipe((f, "dt"), 42) + + reveal_type(answer) # N: Revealed type is "builtins.int" + +- case: test_pipe_tuple_missing_arg + main: | + from xarray import DataTree + + def f(arg: int, dt: DataTree) -> DataTree: + return dt + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are missing an argument for parameter `arg`, so we get no error here. + + dt = DataTree().pipe((f, "dt")) + reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we failed to pass an argument for `arg`. + + dt = DataTree().pipe(lambda data, arg: f(arg, data)) + out: | + main:17: error: No overload variant of "pipe" of "DataTree" matches argument type "Callable[[Any, Any], DataTree]" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T + +- case: test_pipe_tuple_extra_arg + main: | + from xarray import DataTree + + def f(arg: int, dt: DataTree) -> DataTree: + return dt + + # Since we cannot provide a precise type annotation when passing a tuple to + # pipe, there's not enough information for type analysis to indicate that + # we are providing too many args for `f`, so we get no error here. + + dt = DataTree().pipe((f, "dt"), 42, "foo") + reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" + + # Rather than passing a tuple, passing a lambda that calls `f` with args in + # the correct order allows for proper type analysis, indicating (perhaps + # somewhat cryptically) that we passed too many arguments. + + dt = DataTree().pipe(lambda data, arg: f(arg, data), 42, "foo") + out: | + main:17: error: No overload variant of "pipe" of "DataTree" matches argument types "Callable[[Any, Any], DataTree]", "int", "str" [call-overload] + main:17: note: Possible overload variants: + main:17: note: def [P`9, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T + main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T From e28f100b5f26306ab4d744f94ded79952c03cf36 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 11 Feb 2025 14:16:03 -0500 Subject: [PATCH 04/12] Pin mypy to 1.15 in CI --- .github/workflows/ci-additional.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml index b0948dd2e45..d609f7237e7 100644 --- a/.github/workflows/ci-additional.yaml +++ b/.github/workflows/ci-additional.yaml @@ -133,7 +133,7 @@ jobs: python xarray/util/print_versions.py - name: Install mypy run: | - python -m pip install "mypy==1.15" pytest-mypy-plugins --force-reinstall + python -m pip install "mypy==1.15" pytest-mypy-plugins - name: Run mypy run: | From 73f92c9661c73b604ee8f4d54df2b89afde51972 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Wed, 12 Feb 2025 18:57:21 -0500 Subject: [PATCH 05/12] Revert mypy CI job changes --- .github/workflows/ci-additional.yaml | 82 +++++++++++++++++++--------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci-additional.yaml b/.github/workflows/ci-additional.yaml index d609f7237e7..fef104a1ddc 100644 --- a/.github/workflows/ci-additional.yaml +++ b/.github/workflows/ci-additional.yaml @@ -33,15 +33,6 @@ jobs: with: keyword: "[skip-ci]" - detect-act: - name: Detect 'act' runner - runs-on: ubuntu-latest - outputs: - running: ${{ steps.detect-act.outputs.running }} - steps: - - id: detect-act - run: echo "running=${{ env.ACT || 'false' }}" >> "$GITHUB_OUTPUT" - doctest: name: Doctests runs-on: "ubuntu-latest" @@ -90,23 +81,15 @@ jobs: python -m pytest --doctest-modules xarray --ignore xarray/tests -Werror mypy: - strategy: - matrix: - include: - - python-version: "3.10" - codecov-flags: mypy-min - - python-version: "3.12" - codecov-flags: mypy - name: Mypy ${{ matrix.python-version }} + name: Mypy runs-on: "ubuntu-latest" - needs: [detect-ci-trigger, detect-act] - if: always() && (needs.detect-ci-trigger.outputs.triggered == 'true' || needs.detect-act.outputs.running == 'true') + needs: detect-ci-trigger defaults: run: shell: bash -l {0} env: CONDA_ENV_FILE: ci/requirements/environment.yml - PYTHON_VERSION: ${{ matrix.python-version }} + PYTHON_VERSION: "3.12" steps: - uses: actions/checkout@v4 @@ -133,23 +116,68 @@ jobs: python xarray/util/print_versions.py - name: Install mypy run: | - python -m pip install "mypy==1.15" pytest-mypy-plugins + python -m pip install "mypy==1.15" --force-reinstall - name: Run mypy run: | python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report - - name: Run mypy tests - # Run pytest with mypy plugin even if mypy analysis in previous step fails. - if: ${{ always() }} + - name: Upload mypy coverage to Codecov + uses: codecov/codecov-action@v5.3.1 + with: + file: mypy_report/cobertura.xml + flags: mypy + env_vars: PYTHON_VERSION + name: codecov-umbrella + fail_ci_if_error: false + + mypy-min: + name: Mypy 3.10 + runs-on: "ubuntu-latest" + needs: detect-ci-trigger + defaults: + run: + shell: bash -l {0} + env: + CONDA_ENV_FILE: ci/requirements/environment.yml + PYTHON_VERSION: "3.10" + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all branches and tags. + + - name: set environment variables run: | - python -m pytest -v --mypy-only-local-stub --mypy-pyproject-toml-file=pyproject.toml xarray/**/test_*.yml + echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v2 + with: + environment-file: ${{env.CONDA_ENV_FILE}} + environment-name: xarray-tests + create-args: >- + python=${{env.PYTHON_VERSION}} + cache-environment: true + cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" + - name: Install xarray + run: | + python -m pip install --no-deps -e . + - name: Version info + run: | + python xarray/util/print_versions.py + - name: Install mypy + run: | + python -m pip install "mypy==1.15" --force-reinstall + + - name: Run mypy + run: | + python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@v5.3.1 with: - files: mypy_report/cobertura.xml - flags: ${{ matrix.codecov-flags }} + file: mypy_report/cobertura.xml + flags: mypy-min env_vars: PYTHON_VERSION name: codecov-umbrella fail_ci_if_error: false From 5d859b201d64ffedbd48bf1e1210b4d019120848 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Wed, 12 Feb 2025 19:03:07 -0500 Subject: [PATCH 06/12] Add pytest-mypy-plugin and typestub packages --- ci/requirements/environment.yml | 14 +++++++++++++- pyproject.toml | 11 +++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index 321dbe75c38..f1465f5a7e7 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -29,6 +29,7 @@ dependencies: - opt_einsum - packaging - pandas + - pandas-stubs # - pint>=0.22 - pip - pooch @@ -39,14 +40,25 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn - sparse - toolz + - types-colorama + - types-docutils + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools - typing_extensions - zarr - pip: - jax # no way to get cpu-only jaxlib from conda if gpu is present + - types-defusedxml + - types-pexpect diff --git a/pyproject.toml b/pyproject.toml index 32b0bce1322..b1ddbb104f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,12 +40,14 @@ dev = [ "hypothesis", "jinja2", "mypy", + "pandas-stubs", "pre-commit", "pytest", "pytest-cov", "pytest-env", - "pytest-xdist", + "pytest-mypy-plugins", "pytest-timeout", + "pytest-xdist", "ruff>=0.8.0", "sphinx", "sphinx_autosummary_accessors", @@ -304,7 +306,12 @@ known-first-party = ["xarray"] ban-relative-imports = "all" [tool.pytest.ini_options] -addopts = ["--strict-config", "--strict-markers"] +addopts = [ + "--strict-config", + "--strict-markers", + "--mypy-only-local-stub", + "--mypy-pyproject-toml-file=pyproject.toml", +] # We want to forbid warnings from within xarray in our tests — instead we should # fix our own code, or mark the test itself as expecting a warning. So this: From a6f9ef5d5d249e96bebbcb76f4fbc8a7c98a5fa7 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Fri, 14 Feb 2025 11:55:10 -0500 Subject: [PATCH 07/12] Add pytest-mypy-plugins to all conda env files --- .github/workflows/ci.yaml | 13 +++++++++++++ ci/requirements/all-but-dask.yml | 3 ++- ci/requirements/all-but-numba.yml | 3 ++- ci/requirements/bare-minimum.yml | 3 ++- ci/requirements/environment-3.14.yml | 14 +++++++++++++- ci/requirements/environment-windows-3.14.yml | 15 ++++++++++++++- ci/requirements/environment-windows.yml | 15 ++++++++++++++- ci/requirements/min-all-deps.yml | 3 ++- 8 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 713da24f577..1f76af94fea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -143,12 +143,25 @@ jobs: enableCrossOsArchive: true save-always: true + # Run all tests in *.py files, which excludes tests in *.yml files that test type + # annotations via the pytest-mypy-plugins plugin, because the type annotations + # tests fail when using multiple processes (the -n option to pytest). - name: Run tests run: python -m pytest -n 4 --timeout 180 --cov=xarray --cov-report=xml --junitxml=pytest.xml + xarray/tests/test_*.py + + # As noted in the comment on the previous step, we must run type annotation tests + # separately, and we will run them even if the preceding tests failed. Further, + # we must restrict these tests to run only when matrix.env is empty, as this is + # the only case when all of the necessary dependencies are included such that + # spurious mypy errors due to missing packages are eliminated. + - name: Run mypy tests + if: ${{ always() && matrix.env == '' }} + run: python -m pytest xarray/tests/test_*.yml - name: Upload test results if: always() diff --git a/ci/requirements/all-but-dask.yml b/ci/requirements/all-but-dask.yml index b7bf167188f..ca4943bddb1 100644 --- a/ci/requirements/all-but-dask.yml +++ b/ci/requirements/all-but-dask.yml @@ -30,8 +30,9 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn diff --git a/ci/requirements/all-but-numba.yml b/ci/requirements/all-but-numba.yml index 17a657eb32b..fa7ad81f198 100644 --- a/ci/requirements/all-but-numba.yml +++ b/ci/requirements/all-but-numba.yml @@ -43,8 +43,9 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn diff --git a/ci/requirements/bare-minimum.yml b/ci/requirements/bare-minimum.yml index d9590d95165..02e99d34af2 100644 --- a/ci/requirements/bare-minimum.yml +++ b/ci/requirements/bare-minimum.yml @@ -9,8 +9,9 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - numpy=1.24 - packaging=23.1 - pandas=2.1 diff --git a/ci/requirements/environment-3.14.yml b/ci/requirements/environment-3.14.yml index cca3a7a746b..cebae38bc83 100644 --- a/ci/requirements/environment-3.14.yml +++ b/ci/requirements/environment-3.14.yml @@ -29,6 +29,7 @@ dependencies: - opt_einsum - packaging - pandas + - pandas-stubs # - pint>=0.22 - pip - pooch @@ -38,14 +39,25 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn # - sparse - toolz + - types-colorama + - types-docutils + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools - typing_extensions - zarr - pip: - jax # no way to get cpu-only jaxlib from conda if gpu is present + - types-defusedxml + - types-pexpect diff --git a/ci/requirements/environment-windows-3.14.yml b/ci/requirements/environment-windows-3.14.yml index c7f67d2efac..31c91b24b6d 100644 --- a/ci/requirements/environment-windows-3.14.yml +++ b/ci/requirements/environment-windows-3.14.yml @@ -25,6 +25,7 @@ dependencies: - numpy - packaging - pandas + - pandas-stubs # - pint>=0.22 - pip - pre-commit @@ -33,12 +34,24 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn # - sparse - toolz + - types-colorama + - types-docutils + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools - typing_extensions - zarr + - pip: + - types-defusedxml + - types-pexpect diff --git a/ci/requirements/environment-windows.yml b/ci/requirements/environment-windows.yml index a2ecef43d07..f8eb80f6c75 100644 --- a/ci/requirements/environment-windows.yml +++ b/ci/requirements/environment-windows.yml @@ -25,6 +25,7 @@ dependencies: - numpy - packaging - pandas + - pandas-stubs # - pint>=0.22 - pip - pre-commit @@ -33,12 +34,24 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio - scipy - seaborn - sparse - toolz + - types-colorama + - types-docutils + - types-psutil + - types-Pygments + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-setuptools - typing_extensions - zarr + - pip: + - types-defusedxml + - types-pexpect diff --git a/ci/requirements/min-all-deps.yml b/ci/requirements/min-all-deps.yml index f3dab2e5bbf..52c7f9b18e3 100644 --- a/ci/requirements/min-all-deps.yml +++ b/ci/requirements/min-all-deps.yml @@ -46,8 +46,9 @@ dependencies: - pytest - pytest-cov - pytest-env - - pytest-xdist + - pytest-mypy-plugins - pytest-timeout + - pytest-xdist - rasterio=1.3 - scipy=1.11 - seaborn=0.13 From 95084f156cad3201a688abfaa28f4bb1cca54318 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Fri, 14 Feb 2025 12:03:03 -0500 Subject: [PATCH 08/12] Remove dup pandas-stubs dep --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b1ddbb104f9..7c621bffe97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ dev = [ "hypothesis", "jinja2", "mypy", - "pandas-stubs", "pre-commit", "pytest", "pytest-cov", From d92b99275948d53b3f2a668710208b41fd739d25 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 18 Feb 2025 18:33:04 -0500 Subject: [PATCH 09/12] Revert pre-commit config changes --- .pre-commit-config.yaml | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ee39291c3c..f02dbf9dc69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,37 +42,23 @@ repos: - id: prettier args: [--cache-location=.prettier_cache/cache] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.15.0 + rev: v1.14.1 hooks: - id: mypy - files: "^xarray" - exclude: "^xarray/util/generate_.*\\.py$" + # Copied from setup.cfg + exclude: "properties|asv_bench" # This is slow and so we take it out of the fast-path; requires passing # `--hook-stage manual` to pre-commit stages: [manual] - additional_dependencies: - # Type stubs plus additional dependencies from ci/requirements/environment.yml - # required in order to satisfy most (ideally all) type checks. This is rather - # brittle, so it is difficult (if not impossible) to get mypy to succeed in - # this context, even when it succeeds in CI. - - dask - - distributed - - hypothesis - - matplotlib - - numpy==2.1.3 - - pandas-stubs - - pytest - - types-colorama - - types-defusedxml - - types-docutils - - types-pexpect - - types-psutil - - types-Pygments - - types-python-dateutil - - types-pytz - - types-PyYAML - - types-setuptools - - typing-extensions>=4.1.0 + additional_dependencies: [ + # Type stubs + types-python-dateutil, + types-setuptools, + types-PyYAML, + types-pytz, + typing-extensions>=4.1.0, + numpy, + ] - repo: https://github.com/citation-file-format/cff-converter-python rev: ebf0b5e44d67f8beaa1cd13a0d0393ea04c6058d hooks: From a9307cf7cf379877367a48624e0dd0ada914ebe0 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 18 Feb 2025 18:41:48 -0500 Subject: [PATCH 10/12] Place mypy tests behind pytest mypy marker --- .github/workflows/ci.yaml | 30 ++++++++++++++------------ conftest.py | 18 +++++++++++++++- pyproject.toml | 1 + xarray/tests/test_dataarray_typing.yml | 28 ++++++++++++------------ xarray/tests/test_dataset_typing.yml | 28 ++++++++++++------------ xarray/tests/test_datatree_typing.yml | 28 ++++++++++++------------ 6 files changed, 76 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f76af94fea..8061bfe8434 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,6 +49,7 @@ jobs: # Bookend python versions python-version: ["3.10", "3.13"] env: [""] + numprocesses: [4] include: # Minimum python version: - env: "bare-minimum" @@ -67,6 +68,16 @@ jobs: - env: "flaky" python-version: "3.13" os: ubuntu-latest + # The mypy tests must be executed using only 1 process in order to guarantee + # predictable mypy output messages for comparison to expectations. + - env: "mypy" + python-version: "3.10" + numprocesses: 1 + os: ubuntu-latest + - env: "mypy" + python-version: "3.13" + numprocesses: 1 + os: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -88,6 +99,10 @@ jobs: then echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV echo "PYTEST_ADDOPTS=-m 'flaky or network' --run-flaky --run-network-tests -W default" >> $GITHUB_ENV + elif [[ "${{ matrix.env }}" == "mypy" ]] ; + then + echo "CONDA_ENV_FILE=ci/requirements/environment.yml" >> $GITHUB_ENV + echo "PYTEST_ADDOPTS=-n 1 -m 'mypy' --run-mypy -W default" >> $GITHUB_ENV else echo "CONDA_ENV_FILE=ci/requirements/${{ matrix.env }}.yml" >> $GITHUB_ENV fi @@ -143,25 +158,12 @@ jobs: enableCrossOsArchive: true save-always: true - # Run all tests in *.py files, which excludes tests in *.yml files that test type - # annotations via the pytest-mypy-plugins plugin, because the type annotations - # tests fail when using multiple processes (the -n option to pytest). - name: Run tests - run: python -m pytest -n 4 + run: python -m pytest -n ${{ matrix.numprocesses }} --timeout 180 --cov=xarray --cov-report=xml --junitxml=pytest.xml - xarray/tests/test_*.py - - # As noted in the comment on the previous step, we must run type annotation tests - # separately, and we will run them even if the preceding tests failed. Further, - # we must restrict these tests to run only when matrix.env is empty, as this is - # the only case when all of the necessary dependencies are included such that - # spurious mypy errors due to missing packages are eliminated. - - name: Run mypy tests - if: ${{ always() && matrix.env == '' }} - run: python -m pytest xarray/tests/test_*.yml - name: Upload test results if: always() diff --git a/conftest.py b/conftest.py index 24b7530b220..532a7badd91 100644 --- a/conftest.py +++ b/conftest.py @@ -3,7 +3,7 @@ import pytest -def pytest_addoption(parser): +def pytest_addoption(parser: pytest.Parser): """Add command-line flags for pytest.""" parser.addoption("--run-flaky", action="store_true", help="runs flaky tests") parser.addoption( @@ -11,6 +11,7 @@ def pytest_addoption(parser): action="store_true", help="runs tests requiring a network connection", ) + parser.addoption("--run-mypy", action="store_true", help="runs mypy tests") def pytest_runtest_setup(item): @@ -21,6 +22,21 @@ def pytest_runtest_setup(item): pytest.skip( "set --run-network-tests to run test requiring an internet connection" ) + if "mypy" in item.keywords and not item.config.getoption("--run-mypy"): + pytest.skip("set --run-mypy option to run mypy tests") + + +# See https://docs.pytest.org/en/stable/example/markers.html#automatically-adding-markers-based-on-test-names +def pytest_collection_modifyitems(items): + for item in items: + if "mypy" in item.nodeid: + # IMPORTANT: mypy type annotation tests leverage the pytest-mypy-plugins + # plugin, and are thus written in test_*.yml files. As such, there are + # no explicit test functions on which we can apply a pytest.mark.mypy + # decorator. Therefore, we mark them via this name-based, automatic + # marking approach, meaning that each test case must contain "mypy" in the + # name. + item.add_marker(pytest.mark.mypy) @pytest.fixture(autouse=True) diff --git a/pyproject.toml b/pyproject.toml index 7c621bffe97..817fda6c328 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -367,6 +367,7 @@ filterwarnings = [ log_cli_level = "INFO" markers = [ "flaky: flaky tests", + "mypy: type annotation tests", "network: tests requiring a network connection", "slow: slow tests", "slow_hypothesis: slow hypothesis tests", diff --git a/xarray/tests/test_dataarray_typing.yml b/xarray/tests/test_dataarray_typing.yml index 04a6ee452aa..ae3356f9d7c 100644 --- a/xarray/tests/test_dataarray_typing.yml +++ b/xarray/tests/test_dataarray_typing.yml @@ -1,4 +1,4 @@ -- case: test_pipe_lambda_noarg_return_type +- case: test_mypy_pipe_lambda_noarg_return_type main: | from xarray import DataArray @@ -6,7 +6,7 @@ reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" -- case: test_pipe_lambda_posarg_return_type +- case: test_mypy_pipe_lambda_posarg_return_type main: | from xarray import DataArray @@ -14,7 +14,7 @@ reveal_type(da) # N: Revealed type is "builtins.str" -- case: test_pipe_lambda_chaining_return_type +- case: test_mypy_pipe_lambda_chaining_return_type main: | from xarray import DataArray @@ -22,7 +22,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_lambda_missing_arg +- case: test_mypy_pipe_lambda_missing_arg main: | from xarray import DataArray @@ -34,7 +34,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_lambda_extra_arg +- case: test_mypy_pipe_lambda_extra_arg main: | from xarray import DataArray @@ -46,7 +46,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_posarg +- case: test_mypy_pipe_function_missing_posarg main: | from xarray import DataArray @@ -61,7 +61,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_extra_posarg +- case: test_mypy_pipe_function_extra_posarg main: | from xarray import DataArray @@ -76,7 +76,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_kwarg +- case: test_mypy_pipe_function_missing_kwarg main: | from xarray import DataArray @@ -91,7 +91,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_keyword +- case: test_mypy_pipe_function_missing_keyword main: | from xarray import DataArray @@ -106,7 +106,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_unexpected_keyword +- case: test_mypy_pipe_function_unexpected_keyword main: | from xarray import DataArray @@ -118,7 +118,7 @@ out: | main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataWithCoords" [call-arg] -- case: test_pipe_tuple_return_type_dataarray +- case: test_mypy_pipe_tuple_return_type_dataarray main: | from xarray import DataArray @@ -128,7 +128,7 @@ da = DataArray().pipe((f, "da"), 42) reveal_type(da) # N: Revealed type is "xarray.core.dataarray.DataArray" -- case: test_pipe_tuple_return_type_other +- case: test_mypy_pipe_tuple_return_type_other main: | from xarray import DataArray @@ -139,7 +139,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_tuple_missing_arg +- case: test_mypy_pipe_tuple_missing_arg main: | from xarray import DataArray @@ -164,7 +164,7 @@ main:17: note: def [P`9, T] pipe(self, func: Callable[[DataArray, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_tuple_extra_arg +- case: test_mypy_pipe_tuple_extra_arg main: | from xarray import DataArray diff --git a/xarray/tests/test_dataset_typing.yml b/xarray/tests/test_dataset_typing.yml index 99ea4554e84..3b62f81d361 100644 --- a/xarray/tests/test_dataset_typing.yml +++ b/xarray/tests/test_dataset_typing.yml @@ -1,4 +1,4 @@ -- case: test_pipe_lambda_noarg_return_type +- case: test_mypy_pipe_lambda_noarg_return_type main: | from xarray import Dataset @@ -6,7 +6,7 @@ reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" -- case: test_pipe_lambda_posarg_return_type +- case: test_mypy_pipe_lambda_posarg_return_type main: | from xarray import Dataset @@ -14,7 +14,7 @@ reveal_type(ds) # N: Revealed type is "builtins.str" -- case: test_pipe_lambda_chaining_return_type +- case: test_mypy_pipe_lambda_chaining_return_type main: | from xarray import Dataset @@ -22,7 +22,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_lambda_missing_arg +- case: test_mypy_pipe_lambda_missing_arg main: | from xarray import Dataset @@ -34,7 +34,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_lambda_extra_arg +- case: test_mypy_pipe_lambda_extra_arg main: | from xarray import Dataset @@ -46,7 +46,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_posarg +- case: test_mypy_pipe_function_missing_posarg main: | from xarray import Dataset @@ -61,7 +61,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_extra_posarg +- case: test_mypy_pipe_function_extra_posarg main: | from xarray import Dataset @@ -76,7 +76,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_kwarg +- case: test_mypy_pipe_function_missing_kwarg main: | from xarray import Dataset @@ -91,7 +91,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_keyword +- case: test_mypy_pipe_function_missing_keyword main: | from xarray import Dataset @@ -106,7 +106,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_unexpected_keyword +- case: test_mypy_pipe_function_unexpected_keyword main: | from xarray import Dataset @@ -118,7 +118,7 @@ out: | main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataWithCoords" [call-arg] -- case: test_pipe_tuple_return_type_dataset +- case: test_mypy_pipe_tuple_return_type_dataset main: | from xarray import Dataset @@ -128,7 +128,7 @@ ds = Dataset().pipe((f, "ds"), 42) reveal_type(ds) # N: Revealed type is "xarray.core.dataset.Dataset" -- case: test_pipe_tuple_return_type_other +- case: test_mypy_pipe_tuple_return_type_other main: | from xarray import Dataset @@ -139,7 +139,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_tuple_missing_arg +- case: test_mypy_pipe_tuple_missing_arg main: | from xarray import Dataset @@ -164,7 +164,7 @@ main:17: note: def [P`9, T] pipe(self, func: Callable[[Dataset, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_tuple_extra_arg +- case: test_mypy_pipe_tuple_extra_arg main: | from xarray import Dataset diff --git a/xarray/tests/test_datatree_typing.yml b/xarray/tests/test_datatree_typing.yml index 69d566d07e5..fac7fe8ab65 100644 --- a/xarray/tests/test_datatree_typing.yml +++ b/xarray/tests/test_datatree_typing.yml @@ -1,4 +1,4 @@ -- case: test_pipe_lambda_noarg_return_type +- case: test_mypy_pipe_lambda_noarg_return_type main: | from xarray import DataTree @@ -6,7 +6,7 @@ reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" -- case: test_pipe_lambda_posarg_return_type +- case: test_mypy_pipe_lambda_posarg_return_type main: | from xarray import DataTree @@ -14,7 +14,7 @@ reveal_type(dt) # N: Revealed type is "builtins.str" -- case: test_pipe_lambda_chaining_return_type +- case: test_mypy_pipe_lambda_chaining_return_type main: | from xarray import DataTree @@ -22,7 +22,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_lambda_missing_arg +- case: test_mypy_pipe_lambda_missing_arg main: | from xarray import DataTree @@ -34,7 +34,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_lambda_extra_arg +- case: test_mypy_pipe_lambda_extra_arg main: | from xarray import DataTree @@ -46,7 +46,7 @@ main:4: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:4: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_posarg +- case: test_mypy_pipe_function_missing_posarg main: | from xarray import DataTree @@ -61,7 +61,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_extra_posarg +- case: test_mypy_pipe_function_extra_posarg main: | from xarray import DataTree @@ -76,7 +76,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_kwarg +- case: test_mypy_pipe_function_missing_kwarg main: | from xarray import DataTree @@ -91,7 +91,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_missing_keyword +- case: test_mypy_pipe_function_missing_keyword main: | from xarray import DataTree @@ -106,7 +106,7 @@ main:7: note: def [P`2, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:7: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_function_unexpected_keyword +- case: test_mypy_pipe_function_unexpected_keyword main: | from xarray import DataTree @@ -118,7 +118,7 @@ out: | main:7: error: Unexpected keyword argument "kw" for "pipe" of "DataTree" [call-arg] -- case: test_pipe_tuple_return_type_datatree +- case: test_mypy_pipe_tuple_return_type_datatree main: | from xarray import DataTree @@ -128,7 +128,7 @@ dt = DataTree().pipe((f, "dt"), 42) reveal_type(dt) # N: Revealed type is "xarray.core.datatree.DataTree" -- case: test_pipe_tuple_return_type_other +- case: test_mypy_pipe_tuple_return_type_other main: | from xarray import DataTree @@ -139,7 +139,7 @@ reveal_type(answer) # N: Revealed type is "builtins.int" -- case: test_pipe_tuple_missing_arg +- case: test_mypy_pipe_tuple_missing_arg main: | from xarray import DataTree @@ -164,7 +164,7 @@ main:17: note: def [P`9, T] pipe(self, func: Callable[[DataTree, **P], T], *args: P.args, **kwargs: P.kwargs) -> T main:17: note: def [T] pipe(self, func: tuple[Callable[..., T], str], *args: Any, **kwargs: Any) -> T -- case: test_pipe_tuple_extra_arg +- case: test_mypy_pipe_tuple_extra_arg main: | from xarray import DataTree From 83b59face02de474992c57ba6954f613c23aab00 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 18 Feb 2025 18:48:24 -0500 Subject: [PATCH 11/12] Set default pytest numprocesses to 4 --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8061bfe8434..1aad039320a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,7 +49,6 @@ jobs: # Bookend python versions python-version: ["3.10", "3.13"] env: [""] - numprocesses: [4] include: # Minimum python version: - env: "bare-minimum" @@ -159,7 +158,7 @@ jobs: save-always: true - name: Run tests - run: python -m pytest -n ${{ matrix.numprocesses }} + run: python -m pytest -n ${{ matrix.numprocesses || 4 }} --timeout 180 --cov=xarray --cov-report=xml From 6d6083ef3e4077116be1c41dba85b9a463a5d3b4 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Tue, 18 Feb 2025 18:53:44 -0500 Subject: [PATCH 12/12] Ignore pytest-mypy-plugins for min version check --- ci/minimum_versions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/minimum_versions.py b/ci/minimum_versions.py index c226e304769..cc115789d0f 100644 --- a/ci/minimum_versions.py +++ b/ci/minimum_versions.py @@ -26,8 +26,9 @@ "pytest", "pytest-cov", "pytest-env", - "pytest-xdist", + "pytest-mypy-plugins", "pytest-timeout", + "pytest-xdist", "hypothesis", ] pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy