From 1676bddea6db784d9b12865ceb0a08a03ddbf929 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 7 Jul 2025 13:58:37 +0100 Subject: [PATCH 1/4] [mypyc] Speed up for loop over native generator --- mypyc/irbuild/for_helpers.py | 69 ++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index c5b1d1273bef..fd300a7db3c4 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -24,12 +24,15 @@ TypeAlias, ) from mypyc.ir.ops import ( + ERR_NEVER, BasicBlock, Branch, Integer, IntOp, LoadAddress, + LoadErrorValue, LoadMem, + MethodCall, RaiseStandardError, Register, TupleGet, @@ -37,6 +40,7 @@ Value, ) from mypyc.ir.rtypes import ( + RInstance, RTuple, RType, bool_rprimitive, @@ -48,6 +52,8 @@ is_short_int_rprimitive, is_str_rprimitive, is_tuple_rprimitive, + object_pointer_rprimitive, + object_rprimitive, pointer_rprimitive, short_int_rprimitive, ) @@ -62,7 +68,7 @@ dict_next_value_op, dict_value_iter_op, ) -from mypyc.primitives.exc_ops import no_err_occurred_op +from mypyc.primitives.exc_ops import no_err_occurred_op, propagate_if_error_op from mypyc.primitives.generic_ops import aiter_op, anext_op, iter_op, next_op from mypyc.primitives.list_ops import list_append_op, list_get_item_unsafe_op, new_list_set_item_op from mypyc.primitives.misc_ops import stop_async_iteration_op @@ -511,7 +517,16 @@ def make_for_loop_generator( # Default to a generic for loop. if iterable_expr_reg is None: iterable_expr_reg = builder.accept(expr) - for_obj = ForIterable(builder, index, body_block, loop_exit, line, nested) + + helper_method = "__mypyc_generator_helper__" + it = iterable_expr_reg.type + for_obj: ForNativeGenerator | ForIterable + if isinstance(it, RInstance) and it.class_ir.has_method(helper_method): + # Directly call generator object methods if iterating over a native generator. + for_obj = ForNativeGenerator(builder, index, body_block, loop_exit, line, nested) + else: + # Generic implementation that works of arbitrary iterables. + for_obj = ForIterable(builder, index, body_block, loop_exit, line, nested) item_type = builder._analyze_iterable_item_type(expr) item_rtype = builder.type_to_rtype(item_type) for_obj.init(iterable_expr_reg, item_rtype) @@ -623,6 +638,56 @@ def gen_cleanup(self) -> None: self.builder.call_c(no_err_occurred_op, [], self.line) +class ForNativeGenerator(ForGenerator): + """Generate IR for a for loop over a native generator.""" + + def need_cleanup(self) -> bool: + # Create a new cleanup block for when the loop is finished. + return True + + def init(self, expr_reg: Value, target_type: RType) -> None: + # Define targets to contain the expression, along with the iterator that will be used + # for the for-loop. If we are inside of a generator function, spill these into the + # environment class. + builder = self.builder + self.iter_target = builder.maybe_spill(expr_reg) + self.target_type = target_type + + def gen_condition(self) -> None: + builder = self.builder + line = self.line + helper_method = "__mypyc_generator_helper__" + self.return_value = Register(object_rprimitive) + err = builder.add(LoadErrorValue(object_rprimitive, undefines=True)) + builder.assign(self.return_value, err, line) + ptr = builder.add(LoadAddress(object_pointer_rprimitive, self.return_value)) + nn = builder.none_object() + helper_call =( + MethodCall(builder.read(self.iter_target), helper_method, [nn, nn, nn, nn, ptr], line) + ) + # We provide custom handling for error values. + helper_call.error_kind = ERR_NEVER + self.next_reg = builder.add(helper_call) + builder.add(Branch(self.next_reg, self.loop_exit, self.body_block, Branch.IS_ERROR)) + + def begin_body(self) -> None: + # Assign the value obtained from __next__ to the + # lvalue so that it can be referenced by code in the body of the loop. + builder = self.builder + line = self.line + # We unbox here so that iterating with tuple unpacking generates a tuple based + # unpack instead of an iterator based one. + next_reg = builder.coerce(self.next_reg, self.target_type, line) + builder.assign(builder.get_assignment_target(self.index), next_reg, line) + + def gen_step(self) -> None: + # Nothing to do here, since we get the next item as part of gen_condition(). + pass + + def gen_cleanup(self) -> None: + self.builder.primitive_op(propagate_if_error_op, [self.return_value], self.line) + + class ForAsyncIterable(ForGenerator): """Generate IR for an async for loop.""" From a08363f87638c04531dd50db71ff9298623451c9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Jul 2025 14:46:16 +0100 Subject: [PATCH 2/4] Refactor --- mypyc/irbuild/for_helpers.py | 9 ++++----- mypyc/irbuild/prepare.py | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index fd300a7db3c4..893c148e4731 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -58,6 +58,7 @@ short_int_rprimitive, ) from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.targets import AssignmentTarget, AssignmentTargetTuple from mypyc.primitives.dict_ops import ( dict_check_size_op, @@ -518,10 +519,9 @@ def make_for_loop_generator( if iterable_expr_reg is None: iterable_expr_reg = builder.accept(expr) - helper_method = "__mypyc_generator_helper__" it = iterable_expr_reg.type for_obj: ForNativeGenerator | ForIterable - if isinstance(it, RInstance) and it.class_ir.has_method(helper_method): + if isinstance(it, RInstance) and it.class_ir.has_method(GENERATOR_HELPER_NAME): # Directly call generator object methods if iterating over a native generator. for_obj = ForNativeGenerator(builder, index, body_block, loop_exit, line, nested) else: @@ -656,14 +656,13 @@ def init(self, expr_reg: Value, target_type: RType) -> None: def gen_condition(self) -> None: builder = self.builder line = self.line - helper_method = "__mypyc_generator_helper__" self.return_value = Register(object_rprimitive) err = builder.add(LoadErrorValue(object_rprimitive, undefines=True)) builder.assign(self.return_value, err, line) ptr = builder.add(LoadAddress(object_pointer_rprimitive, self.return_value)) nn = builder.none_object() - helper_call =( - MethodCall(builder.read(self.iter_target), helper_method, [nn, nn, nn, nn, ptr], line) + helper_call = MethodCall( + builder.read(self.iter_target), GENERATOR_HELPER_NAME, [nn, nn, nn, nn, ptr], line ) # We provide custom handling for error values. helper_call.error_kind = ERR_NEVER diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index c22101ac193d..4eff90f90b7d 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -15,7 +15,7 @@ from collections import defaultdict from collections.abc import Iterable -from typing import NamedTuple +from typing import Final, NamedTuple from mypy.build import Graph from mypy.nodes import ( @@ -71,7 +71,7 @@ from mypyc.options import CompilerOptions from mypyc.sametype import is_same_type -GENERATOR_HELPER_NAME = "__mypyc_generator_helper__" +GENERATOR_HELPER_NAME: Final = "__mypyc_generator_helper__" def build_type_map( From 27656a305aae4dea4d9b5af4f397a99509f79bf9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Jul 2025 14:50:03 +0100 Subject: [PATCH 3/4] Add comments --- mypyc/irbuild/for_helpers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 893c148e4731..73228ceab02c 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -659,6 +659,10 @@ def gen_condition(self) -> None: self.return_value = Register(object_rprimitive) err = builder.add(LoadErrorValue(object_rprimitive, undefines=True)) builder.assign(self.return_value, err, line) + + # Call generated generator helper method, passing a PyObject ** as the final + # argument that will be used to store the return value in this register. This + # is faster than raising StopIteration. ptr = builder.add(LoadAddress(object_pointer_rprimitive, self.return_value)) nn = builder.none_object() helper_call = MethodCall( @@ -666,6 +670,7 @@ def gen_condition(self) -> None: ) # We provide custom handling for error values. helper_call.error_kind = ERR_NEVER + self.next_reg = builder.add(helper_call) builder.add(Branch(self.next_reg, self.loop_exit, self.body_block, Branch.IS_ERROR)) @@ -684,6 +689,8 @@ def gen_step(self) -> None: pass def gen_cleanup(self) -> None: + # If return value is NULL (it wasn't assigned to by the generator helper method), + # an exception was raised. self.builder.primitive_op(propagate_if_error_op, [self.return_value], self.line) From 9dec2079262523248377d3bc783e142265b86f7d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Jul 2025 15:45:40 +0100 Subject: [PATCH 4/4] Update comments --- mypyc/irbuild/for_helpers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 73228ceab02c..34d4e9f56b5f 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -646,9 +646,8 @@ def need_cleanup(self) -> bool: return True def init(self, expr_reg: Value, target_type: RType) -> None: - # Define targets to contain the expression, along with the iterator that will be used - # for the for-loop. If we are inside of a generator function, spill these into the - # environment class. + # Define target to contains the generator expression. It's also the iterator. + # If we are inside a generator function, spill these into the environment class. builder = self.builder self.iter_target = builder.maybe_spill(expr_reg) self.target_type = target_type @@ -661,8 +660,10 @@ def gen_condition(self) -> None: builder.assign(self.return_value, err, line) # Call generated generator helper method, passing a PyObject ** as the final - # argument that will be used to store the return value in this register. This - # is faster than raising StopIteration. + # argument that will be used to store the return value in the return value + # register. We ignore the return value but the presence of a return value + # indicates that the generator has finished. This is faster than raising + # and catching StopIteration, which is the non-native way of doing this. ptr = builder.add(LoadAddress(object_pointer_rprimitive, self.return_value)) nn = builder.none_object() helper_call = MethodCall( @@ -675,7 +676,7 @@ def gen_condition(self) -> None: builder.add(Branch(self.next_reg, self.loop_exit, self.body_block, Branch.IS_ERROR)) def begin_body(self) -> None: - # Assign the value obtained from __next__ to the + # Assign the value obtained from the generator helper method to the # lvalue so that it can be referenced by code in the body of the loop. builder = self.builder line = self.line @@ -690,7 +691,7 @@ def gen_step(self) -> None: def gen_cleanup(self) -> None: # If return value is NULL (it wasn't assigned to by the generator helper method), - # an exception was raised. + # an exception was raised that we need to propagate. self.builder.primitive_op(propagate_if_error_op, [self.return_value], self.line) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy