From 3010efc873f0dfd0eef2c971feee1f9cc2ce97d8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 24 Mar 2025 20:08:57 +0000 Subject: [PATCH 1/3] Consolidate descriptor handling in checkmember.py (#18831) This is not a pure refactoring, but almost. Right now we are in a weird situation where we have two inconsistencies: * `__set__()` is handled in `checker.py` while `__get__()` is handled in `checkmember.py` * rules for when to use binder are slightly different between descriptors and settable properties. This PR fixes these two things. As a nice bonus we should get free support for unions in `__set__()`. --- mypy/checker.py | 24 ++++++--- mypy/checkexpr.py | 10 +++- mypy/checkmember.py | 126 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 133 insertions(+), 27 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index ac4b24709783..fb435870df8c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3170,7 +3170,7 @@ def check_assignment( ) else: self.try_infer_partial_generic_type_from_assignment(lvalue, rvalue, "=") - lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue) + lvalue_type, index_lvalue, inferred = self.check_lvalue(lvalue, rvalue) # If we're assigning to __getattr__ or similar methods, check that the signature is # valid. if isinstance(lvalue, NameExpr) and lvalue.node: @@ -4263,7 +4263,9 @@ def check_multi_assignment_from_iterable( else: self.msg.type_not_iterable(rvalue_type, context) - def check_lvalue(self, lvalue: Lvalue) -> tuple[Type | None, IndexExpr | None, Var | None]: + def check_lvalue( + self, lvalue: Lvalue, rvalue: Expression | None = None + ) -> tuple[Type | None, IndexExpr | None, Var | None]: lvalue_type = None index_lvalue = None inferred = None @@ -4281,7 +4283,7 @@ def check_lvalue(self, lvalue: Lvalue) -> tuple[Type | None, IndexExpr | None, V elif isinstance(lvalue, IndexExpr): index_lvalue = lvalue elif isinstance(lvalue, MemberExpr): - lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, True) + lvalue_type = self.expr_checker.analyze_ordinary_member_access(lvalue, True, rvalue) self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, NameExpr): lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) @@ -4552,12 +4554,8 @@ def check_member_assignment( Return the inferred rvalue_type, inferred lvalue_type, and whether to use the binder for this assignment. - - Note: this method exists here and not in checkmember.py, because we need to take - care about interaction between binder and __set__(). """ instance_type = get_proper_type(instance_type) - attribute_type = get_proper_type(attribute_type) # Descriptors don't participate in class-attribute access if (isinstance(instance_type, FunctionLike) and instance_type.is_type_obj()) or isinstance( instance_type, TypeType @@ -4569,8 +4567,8 @@ def check_member_assignment( get_lvalue_type = self.expr_checker.analyze_ordinary_member_access( lvalue, is_lvalue=False ) - use_binder = is_same_type(get_lvalue_type, attribute_type) +<<<<<<< HEAD if not isinstance(attribute_type, Instance): # TODO: support __set__() for union types. rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context) @@ -4664,13 +4662,23 @@ def check_member_assignment( return AnyType(TypeOfAny.from_error), get_type, False set_type = inferred_dunder_set_type.arg_types[1] +======= +>>>>>>> df9ddfcac (Consolidate descriptor handling in checkmember.py (#18831)) # Special case: if the rvalue_type is a subtype of both '__get__' and '__set__' types, # and '__get__' type is narrower than '__set__', then we invoke the binder to narrow type # by this assignment. Technically, this is not safe, but in practice this is # what a user expects. +<<<<<<< HEAD rvalue_type = self.check_simple_assignment(set_type, rvalue, context) infer = is_subtype(rvalue_type, get_type) and is_subtype(get_type, set_type) return rvalue_type if infer else set_type, get_type, infer +======= + rvalue_type, _ = self.check_simple_assignment(attribute_type, rvalue, context) + infer = is_subtype(rvalue_type, get_lvalue_type) and is_subtype( + get_lvalue_type, attribute_type + ) + return rvalue_type if infer else attribute_type, attribute_type, infer +>>>>>>> df9ddfcac (Consolidate descriptor handling in checkmember.py (#18831)) def check_indexed_assignment( self, lvalue: IndexExpr, rvalue: Expression, context: Context diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 1017009ce7ab..3fa4df2a7171 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3327,8 +3327,13 @@ def visit_member_expr(self, e: MemberExpr, is_lvalue: bool = False) -> Type: self.chk.warn_deprecated(e.node, e) return narrowed - def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type: - """Analyse member expression or member lvalue.""" + def analyze_ordinary_member_access( + self, e: MemberExpr, is_lvalue: bool, rvalue: Expression | None = None + ) -> Type: + """Analyse member expression or member lvalue. + + An rvalue can be provided optionally to infer better setter type when is_lvalue is True. + """ if e.kind is not None: # This is a reference to a module attribute. return self.analyze_ref_expr(e) @@ -3360,6 +3365,7 @@ def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type in_literal_context=self.is_literal_context(), module_symbol_table=module_symbol_table, is_self=is_self, + rvalue=rvalue, ) return member_type diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 0994d0df400b..9b7c8027e1ed 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -23,6 +23,7 @@ ArgKind, Context, Decorator, + Expression, FuncBase, FuncDef, IndexExpr, @@ -101,6 +102,7 @@ def __init__( module_symbol_table: SymbolTable | None = None, no_deferral: bool = False, is_self: bool = False, + rvalue: Expression | None = None, ) -> None: self.is_lvalue = is_lvalue self.is_super = is_super @@ -113,6 +115,9 @@ def __init__( self.module_symbol_table = module_symbol_table self.no_deferral = no_deferral self.is_self = is_self + if rvalue is not None: + assert is_lvalue + self.rvalue = rvalue def named_type(self, name: str) -> Instance: return self.chk.named_type(name) @@ -139,6 +144,7 @@ def copy_modified( self_type=self.self_type, module_symbol_table=self.module_symbol_table, no_deferral=self.no_deferral, + rvalue=self.rvalue, ) if messages is not None: mx.msg = messages @@ -168,6 +174,7 @@ def analyze_member_access( module_symbol_table: SymbolTable | None = None, no_deferral: bool = False, is_self: bool = False, + rvalue: Expression | None = None, ) -> Type: """Return the type of attribute 'name' of 'typ'. @@ -186,11 +193,14 @@ def analyze_member_access( of 'original_type'. 'original_type' is always preserved as the 'typ' type used in the initial, non-recursive call. The 'self_type' is a component of 'original_type' to which generic self should be bound (a narrower type that has a fallback to instance). - Currently this is used only for union types. + Currently, this is used only for union types. - 'module_symbol_table' is passed to this function if 'typ' is actually a module + 'module_symbol_table' is passed to this function if 'typ' is actually a module, and we want to keep track of the available attributes of the module (since they are not available via the type object directly) + + 'rvalue' can be provided optionally to infer better setter type when is_lvalue is True, + most notably this helps for descriptors with overloaded __set__() method. """ mx = MemberContext( is_lvalue=is_lvalue, @@ -204,6 +214,7 @@ def analyze_member_access( module_symbol_table=module_symbol_table, no_deferral=no_deferral, is_self=is_self, + rvalue=rvalue, ) result = _analyze_member_access(name, typ, mx, override_info) possible_literal = get_proper_type(result) @@ -629,9 +640,7 @@ def check_final_member(name: str, info: TypeInfo, msg: MessageBuilder, ctx: Cont msg.cant_assign_to_final(name, attr_assign=True, ctx=ctx) -def analyze_descriptor_access( - descriptor_type: Type, mx: MemberContext, *, assignment: bool = False -) -> Type: +def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: """Type check descriptor access. Arguments: @@ -639,7 +648,7 @@ def analyze_descriptor_access( (the type of ``f`` in ``a.f`` when ``f`` is a descriptor). mx: The current member access context. Return: - The return type of the appropriate ``__get__`` overload for the descriptor. + The return type of the appropriate ``__get__/__set__`` overload for the descriptor. """ instance_type = get_proper_type(mx.self_type) orig_descriptor_type = descriptor_type @@ -648,15 +657,24 @@ def analyze_descriptor_access( if isinstance(descriptor_type, UnionType): # Map the access over union types return make_simplified_union( - [ - analyze_descriptor_access(typ, mx, assignment=assignment) - for typ in descriptor_type.items - ] + [analyze_descriptor_access(typ, mx) for typ in descriptor_type.items] ) elif not isinstance(descriptor_type, Instance): return orig_descriptor_type - if not descriptor_type.type.has_readable_member("__get__"): + if not mx.is_lvalue and not descriptor_type.type.has_readable_member("__get__"): + return orig_descriptor_type + + # We do this check first to accommodate for descriptors with only __set__ method. + # If there is no __set__, we type-check that the assigned value matches + # the return type of __get__. This doesn't match the python semantics, + # (which allow you to override the descriptor with any value), but preserves + # the type of accessing the attribute (even after the override). + if mx.is_lvalue and descriptor_type.type.has_readable_member("__set__"): + return analyze_descriptor_assign(descriptor_type, mx) + + if mx.is_lvalue and not descriptor_type.type.has_readable_member("__get__"): + # This turned out to be not a descriptor after all. return orig_descriptor_type dunder_get = descriptor_type.type.get_method("__get__") @@ -713,11 +731,10 @@ def analyze_descriptor_access( callable_name=callable_name, ) - if not assignment: - mx.chk.check_deprecated(dunder_get, mx.context) - mx.chk.warn_deprecated_overload_item( - dunder_get, mx.context, target=inferred_dunder_get_type, selftype=descriptor_type - ) + mx.chk.check_deprecated(dunder_get, mx.context) + mx.chk.warn_deprecated_overload_item( + dunder_get, mx.context, target=inferred_dunder_get_type, selftype=descriptor_type + ) inferred_dunder_get_type = get_proper_type(inferred_dunder_get_type) if isinstance(inferred_dunder_get_type, AnyType): @@ -736,6 +753,79 @@ def analyze_descriptor_access( return inferred_dunder_get_type.ret_type +def analyze_descriptor_assign(descriptor_type: Instance, mx: MemberContext) -> Type: + instance_type = get_proper_type(mx.self_type) + dunder_set = descriptor_type.type.get_method("__set__") + if dunder_set is None: + mx.chk.fail( + message_registry.DESCRIPTOR_SET_NOT_CALLABLE.format( + descriptor_type.str_with_options(mx.msg.options) + ), + mx.context, + ) + return AnyType(TypeOfAny.from_error) + + bound_method = analyze_decorator_or_funcbase_access( + defn=dunder_set, + itype=descriptor_type, + name="__set__", + mx=mx.copy_modified(is_lvalue=False, self_type=descriptor_type), + ) + typ = map_instance_to_supertype(descriptor_type, dunder_set.info) + dunder_set_type = expand_type_by_instance(bound_method, typ) + + callable_name = mx.chk.expr_checker.method_fullname(descriptor_type, "__set__") + rvalue = mx.rvalue or TempNode(AnyType(TypeOfAny.special_form), context=mx.context) + dunder_set_type = mx.chk.expr_checker.transform_callee_type( + callable_name, + dunder_set_type, + [TempNode(instance_type, context=mx.context), rvalue], + [ARG_POS, ARG_POS], + mx.context, + object_type=descriptor_type, + ) + + # For non-overloaded setters, the result should be type-checked like a regular assignment. + # Hence, we first only try to infer the type by using the rvalue as type context. + type_context = rvalue + with mx.msg.filter_errors(): + _, inferred_dunder_set_type = mx.chk.expr_checker.check_call( + dunder_set_type, + [TempNode(instance_type, context=mx.context), type_context], + [ARG_POS, ARG_POS], + mx.context, + object_type=descriptor_type, + callable_name=callable_name, + ) + + # And now we in fact type check the call, to show errors related to wrong arguments + # count, etc., replacing the type context for non-overloaded setters only. + inferred_dunder_set_type = get_proper_type(inferred_dunder_set_type) + if isinstance(inferred_dunder_set_type, CallableType): + type_context = TempNode(AnyType(TypeOfAny.special_form), context=mx.context) + mx.chk.expr_checker.check_call( + dunder_set_type, + [TempNode(instance_type, context=mx.context), type_context], + [ARG_POS, ARG_POS], + mx.context, + object_type=descriptor_type, + callable_name=callable_name, + ) + + # Search for possible deprecations: + mx.chk.check_deprecated(dunder_set, mx.context) + mx.chk.warn_deprecated_overload_item( + dunder_set, mx.context, target=inferred_dunder_set_type, selftype=descriptor_type + ) + + # In the following cases, a message already will have been recorded in check_call. + if (not isinstance(inferred_dunder_set_type, CallableType)) or ( + len(inferred_dunder_set_type.arg_types) < 2 + ): + return AnyType(TypeOfAny.from_error) + return inferred_dunder_set_type.arg_types[1] + + def is_instance_var(var: Var) -> bool: """Return if var is an instance variable according to PEP 526.""" return ( @@ -820,6 +910,7 @@ def analyze_var( # A property cannot have an overloaded type => the cast is fine. assert isinstance(expanded_signature, CallableType) if var.is_settable_property and mx.is_lvalue and var.setter_type is not None: + # TODO: use check_call() to infer better type, same as for __set__(). result = expanded_signature.arg_types[0] else: result = expanded_signature.ret_type @@ -832,7 +923,7 @@ def analyze_var( result = AnyType(TypeOfAny.special_form) fullname = f"{var.info.fullname}.{name}" hook = mx.chk.plugin.get_attribute_hook(fullname) - if result and not mx.is_lvalue and not implicit: + if result and not (implicit or var.info.is_protocol and is_instance_var(var)): result = analyze_descriptor_access(result, mx) if hook: result = hook( @@ -1106,6 +1197,7 @@ def analyze_class_attribute_access( result = add_class_tvars( t, isuper, is_classmethod, is_staticmethod, mx.self_type, original_vars=original_vars ) + # __set__ is not called on class objects. if not mx.is_lvalue: result = analyze_descriptor_access(result, mx) From 8e7c094a409f766395f2c4c31e0edbf99614df31 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 30 Apr 2025 10:20:34 +0100 Subject: [PATCH 2/3] Local forward refs should precede global forward refs (#19000) Fixes https://github.com/python/mypy/issues/18988 This should be a minimal change to restore backwards compatibility for an edge case with forward references. --- mypy/semanal.py | 9 ++++++++ test-data/unit/check-python312.test | 36 ++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a0cfdcce1e33..a3e113252ac3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -6336,6 +6336,8 @@ class C: if node.name not in self.globals: return True global_node = self.globals[node.name] + if not self.is_textually_before_class(global_node.node): + return True return not self.is_type_like(global_node.node) return False @@ -6363,6 +6365,13 @@ def is_textually_before_statement(self, node: SymbolNode) -> bool: else: return line_diff > 0 + def is_textually_before_class(self, node: SymbolNode | None) -> bool: + """Similar to above, but check if a node is defined before current class.""" + assert self.type is not None + if node is None: + return False + return node.line < self.type.defn.line + def is_overloaded_item(self, node: SymbolNode, statement: Statement) -> bool: """Check whether the function belongs to the overloaded variants""" if isinstance(node, OverloadedFuncDef) and isinstance(statement, FuncDef): diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index ba4104a50048..2244548ea969 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -2004,15 +2004,18 @@ reveal_type(x.related_resources) # N: Revealed type is "__main__.ResourceRule" [case testPEP695TypeAliasRecursiveOuterClass] class A: - type X = X + type X = X # E: Cannot resolve name "X" (possible cyclic definition) class X: ... +class AA: + XX = XX # OK, we allow this as a special case. +class XX: ... + class Y: ... class B: type Y = Y -x: A.X -reveal_type(x) # N: Revealed type is "__main__.X" +reveal_type(AA.XX) # N: Revealed type is "def () -> __main__.XX" y: B.Y reveal_type(y) # N: Revealed type is "__main__.Y" [builtins fixtures/tuple.pyi] @@ -2029,3 +2032,30 @@ def foo() -> None: class Z: ... # E: Name "Z" already defined on line 2 [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695MultipleUnpacksInBareApplicationNoCrash] +# https://github.com/python/mypy/issues/18856 +class A[*Ts]: ... + +A[*tuple[int, ...], *tuple[int, ...]] # E: More than one Unpack in a type is not allowed +a: A[*tuple[int, ...], *tuple[int, ...]] # E: More than one Unpack in a type is not allowed +def foo(a: A[*tuple[int, ...], *tuple[int, ...]]): ... # E: More than one Unpack in a type is not allowed + +tuple[*tuple[int, ...], *tuple[int, ...]] # E: More than one Unpack in a type is not allowed +b: tuple[*tuple[int, ...], *tuple[int, ...]] # E: More than one Unpack in a type is not allowed +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testForwardNestedPrecedesForwardGlobal] +from typing import NewType + +class W[T]: pass + +class R: + class M(W[Action.V], type): + FOO = R.Action.V(0) + class Action(metaclass=M): + V = NewType('V', int) + +class Action: + pass From ea8aacd27d8d7252ee88b5715889be03e0e88c3f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 30 Apr 2025 16:35:27 +0100 Subject: [PATCH 3/3] Do not narrow types to Never with binder (#18972) Fixes https://github.com/python/mypy/issues/18967 Fixes https://github.com/python/mypy/issues/16494 Fixes https://github.com/python/mypy/issues/15793 Fixes https://github.com/python/mypy/issues/12949 As you can see from updated test cases, it is kind of gray area, so whether we go this way will depend on the `mypy_primer` results (and also potentially on Dropbox internal code bases, where the above issue may cause problems). --- mypy/checkexpr.py | 8 +++++++- test-data/unit/check-isinstance.test | 4 ++-- test-data/unit/check-narrowing.test | 13 ++++++++++++- test-data/unit/check-python310.test | 2 +- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 3fa4df2a7171..008e056f4367 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -6277,7 +6277,13 @@ def narrow_type_from_binder( known_type, restriction, prohibit_none_typevar_overlap=True ): return None - return narrow_declared_type(known_type, restriction) + narrowed = narrow_declared_type(known_type, restriction) + if isinstance(get_proper_type(narrowed), UninhabitedType): + # If we hit this case, it means that we can't reliably mark the code as + # unreachable, but the resulting type can't be expressed in type system. + # Falling back to restriction is more intuitive in most cases. + return restriction + return narrowed return known_type def has_abstract_type_part(self, caller_type: ProperType, callee_type: ProperType) -> bool: diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 49140bf52b8d..058db1ea8197 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -1812,9 +1812,9 @@ reveal_type(fm) # N: Revealed type is "__main__.FooMetaclass" if issubclass(fm, Foo): reveal_type(fm) # N: Revealed type is "Type[__main__.Foo]" if issubclass(fm, Bar): - reveal_type(fm) # N: Revealed type is "Never" + reveal_type(fm) # N: Revealed type is "Type[__main__.Bar]" if issubclass(fm, Baz): - reveal_type(fm) # N: Revealed type is "Never" + reveal_type(fm) # N: Revealed type is "Type[__main__.Baz]" [builtins fixtures/isinstance.pyi] [case testIsinstanceAndNarrowTypeVariable] diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 1856ca26f736..dc2cfd46d9ad 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -1284,7 +1284,7 @@ def f(t: Type[T], a: A, b: B) -> None: reveal_type(a) # N: Revealed type is "__main__.A" if type(b) is t: - reveal_type(b) # N: Revealed type is "Never" + reveal_type(b) # N: Revealed type is "T`-1" else: reveal_type(b) # N: Revealed type is "__main__.B" @@ -2413,3 +2413,14 @@ def foo(x: T) -> T: reveal_type(x) # N: Revealed type is "T`-1" return x [builtins fixtures/isinstance.pyi] + +[case testDoNotNarrowToNever] +def any(): + return 1 + +def f() -> None: + x = "a" + x = any() + assert isinstance(x, int) + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstance.pyi] diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 016f50552a5f..a25a7b7107c7 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1299,7 +1299,7 @@ m: str match m: case a if a := 1: # E: Incompatible types in assignment (expression has type "int", variable has type "str") - reveal_type(a) # N: Revealed type is "Never" + reveal_type(a) # N: Revealed type is "Literal[1]?" [case testMatchAssigningPatternGuard] m: str pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy