From 7476442c6dab738226715823b813cd1601a6683b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Apr 2025 09:43:08 +0100 Subject: [PATCH 01/21] FOR_ITER now pushes NULL as well as the iterator. Preparation for virtual iterators --- Include/internal/pycore_compile.h | 1 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 38 ++-- Include/internal/pycore_uop_ids.h | 6 + Include/internal/pycore_uop_metadata.h | 4 + Lib/test/test_dis.py | 205 +++++++++++----------- Objects/frameobject.c | 4 + Programs/test_frozenmain.h | 2 +- Python/bytecodes.c | 54 +++--- Python/codegen.c | 39 ++-- Python/executor_cases.c.h | 43 +++-- Python/flowgraph.c | 28 ++- Python/generated_cases.c.h | 39 ++-- Python/optimizer_cases.c.h | 11 ++ 14 files changed, 276 insertions(+), 201 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index a606c2afe0a234..dbf565eb5d7623 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -95,6 +95,7 @@ typedef enum { enum _PyCompile_FBlockType { COMPILE_FBLOCK_WHILE_LOOP, COMPILE_FBLOCK_FOR_LOOP, + COMPILE_FBLOCK_ASYNC_FOR_LOOP, COMPILE_FBLOCK_TRY_EXCEPT, COMPILE_FBLOCK_FINALLY_TRY, COMPILE_FBLOCK_FINALLY_END, diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index f75b05893affc1..668d93a0609b1f 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -275,6 +275,7 @@ Known values: Python 3.14a6 3620 (Optimize bytecode for all/any/tuple called on a genexp) Python 3.14a7 3621 (Optimize LOAD_FAST opcodes into LOAD_FAST_BORROW) Python 3.14a7 3622 (Store annotations in different class dict keys) + Python 3.14a8 3623 (Virtual iterators) Python 3.15 will start with 3650 @@ -287,7 +288,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3622 +#define PYC_MAGIC_NUMBER 3623 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3b881b30b054f1..44bd6de9da6740 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -199,15 +199,15 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case FORMAT_WITH_SPEC: return 2; case FOR_ITER: - return 1; + return 2; case FOR_ITER_GEN: - return 1; + return 2; case FOR_ITER_LIST: - return 1; + return 2; case FOR_ITER_RANGE: - return 1; + return 2; case FOR_ITER_TUPLE: - return 1; + return 2; case GET_AITER: return 1; case GET_ANEXT: @@ -233,11 +233,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_END_ASYNC_FOR: return 2; case INSTRUMENTED_END_FOR: - return 2; + return 3; case INSTRUMENTED_END_SEND: return 2; case INSTRUMENTED_FOR_ITER: - return 1; + return 2; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_BACKWARD: @@ -251,7 +251,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_NOT_TAKEN: return 0; case INSTRUMENTED_POP_ITER: - return 1; + return 2; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 1; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -393,7 +393,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case POP_EXCEPT: return 1; case POP_ITER: - return 1; + return 2; case POP_JUMP_IF_FALSE: return 1; case POP_JUMP_IF_NONE: @@ -680,15 +680,15 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case FORMAT_WITH_SPEC: return 1; case FOR_ITER: - return 2; + return 3; case FOR_ITER_GEN: - return 1; - case FOR_ITER_LIST: return 2; + case FOR_ITER_LIST: + return 3; case FOR_ITER_RANGE: - return 2; + return 3; case FOR_ITER_TUPLE: - return 2; + return 3; case GET_AITER: return 1; case GET_ANEXT: @@ -696,7 +696,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_AWAITABLE: return 1; case GET_ITER: - return 1; + return 2; case GET_LEN: return 2; case GET_YIELD_FROM_ITER: @@ -714,11 +714,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case INSTRUMENTED_END_ASYNC_FOR: return 0; case INSTRUMENTED_END_FOR: - return 1; + return 2; case INSTRUMENTED_END_SEND: return 1; case INSTRUMENTED_FOR_ITER: - return 2; + return 3; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_BACKWARD: @@ -1237,7 +1237,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1446,7 +1446,7 @@ _PyOpcode_macro_expansion[256] = { [NOP] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, OPARG_SIMPLE, 0 } } }, - [POP_ITER] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [POP_ITER] = { .nuops = 1, .uops = { { _POP_ITER, OPARG_SIMPLE, 0 } } }, [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_TRUE, OPARG_REPLACED, 1 } } }, [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d6d81f88c8e00c..3155b299f3db4b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -256,8 +256,14 @@ extern "C" { #define _MONITOR_RESUME 471 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT +<<<<<<< HEAD #define _POP_JUMP_IF_FALSE 472 #define _POP_JUMP_IF_TRUE 473 +======= +#define _POP_ITER POP_ITER +#define _POP_JUMP_IF_FALSE 465 +#define _POP_JUMP_IF_TRUE 466 +>>>>>>> d908cf7e4c1 (FOR_ITER now pushes NULL as well as the iterator. Preparation for virtual iterators) #define _POP_TOP POP_TOP #define _POP_TOP_LOAD_CONST_INLINE 474 #define _POP_TOP_LOAD_CONST_INLINE_BORROW 475 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0ea8e56b6cd0bc..c339888b892f5b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -66,6 +66,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, + [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, @@ -534,6 +535,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", [_NOP] = "_NOP", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_ITER] = "_POP_ITER", [_POP_TOP] = "_POP_TOP", [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", @@ -694,6 +696,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _END_FOR: return 1; + case _POP_ITER: + return 2; case _END_SEND: return 2; case _UNARY_NEGATIVE: diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f2586fcee57d87..8495c6b938c845 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1821,7 +1821,7 @@ def _prepare_test_cases(): make_inst(opname='LOAD_SMALL_INT', arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3), make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='GET_ITER', arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3), - make_inst(opname='FOR_ITER', arg=32, argval=92, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='FOR_ITER', arg=33, argval=94, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4), @@ -1840,110 +1840,111 @@ def _prepare_test_cases(): make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=82, start_offset=82, starts_line=False, line_number=7), make_inst(opname='JUMP_BACKWARD', arg=32, argval=24, argrepr='to L1', offset=84, start_offset=84, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=8, label=3), - make_inst(opname='JUMP_FORWARD', arg=13, argval=118, argrepr='to L5', offset=90, start_offset=90, starts_line=False, line_number=8), - make_inst(opname='END_FOR', arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=3, label=4), - make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=False, line_number=3), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=96, start_offset=96, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=106, start_offset=106, starts_line=False, line_number=10), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=108, start_offset=108, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=10), - make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=118, start_offset=118, starts_line=True, line_number=11, label=5), - make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_JUMP_IF_FALSE', arg=40, argval=212, argrepr='to L8', offset=128, start_offset=128, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=11), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=134, start_offset=134, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=144, start_offset=144, starts_line=False, line_number=12), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=154, start_offset=154, starts_line=False, line_number=12), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=True, line_number=13), - make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=13), - make_inst(opname='BINARY_OP', arg=23, argval=23, argrepr='-=', offset=160, start_offset=160, starts_line=False, line_number=13, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), - make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=172, start_offset=172, starts_line=False, line_number=13), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=14), - make_inst(opname='LOAD_SMALL_INT', arg=6, argval=6, argrepr='', offset=176, start_offset=176, starts_line=False, line_number=14), - make_inst(opname='COMPARE_OP', arg=148, argval='>', argrepr='bool(>)', offset=178, start_offset=178, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='POP_JUMP_IF_FALSE', arg=3, argval=192, argrepr='to L6', offset=182, start_offset=182, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=186, start_offset=186, starts_line=False, line_number=14), - make_inst(opname='JUMP_BACKWARD', arg=37, argval=118, argrepr='to L5', offset=188, start_offset=188, starts_line=True, line_number=15, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=192, start_offset=192, starts_line=True, line_number=16, label=6), - make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=194, start_offset=194, starts_line=False, line_number=16), - make_inst(opname='COMPARE_OP', arg=18, argval='<', argrepr='bool(<)', offset=196, start_offset=196, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='POP_JUMP_IF_TRUE', arg=3, argval=210, argrepr='to L7', offset=200, start_offset=200, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=204, start_offset=204, starts_line=False, line_number=16), - make_inst(opname='JUMP_BACKWARD', arg=46, argval=118, argrepr='to L5', offset=206, start_offset=206, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='JUMP_FORWARD', arg=11, argval=234, argrepr='to L9', offset=210, start_offset=210, starts_line=True, line_number=17, label=7), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=212, start_offset=212, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=222, start_offset=222, starts_line=False, line_number=19), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=224, start_offset=224, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=False, line_number=19), - make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=20, label=9), - make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=21), - make_inst(opname='LOAD_SMALL_INT', arg=0, argval=0, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21), - make_inst(opname='BINARY_OP', arg=11, argval=11, argrepr='/', offset=240, start_offset=240, starts_line=False, line_number=21, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=21), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=254, start_offset=254, starts_line=True, line_number=25), - make_inst(opname='COPY', arg=1, argval=1, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25), - make_inst(opname='LOAD_SPECIAL', arg=1, argval=1, argrepr='__exit__', offset=258, start_offset=258, starts_line=False, line_number=25), - make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=25), - make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25), - make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=264, start_offset=264, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=274, start_offset=274, starts_line=False, line_number=25), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=276, start_offset=276, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=286, start_offset=286, starts_line=False, line_number=26), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=288, start_offset=288, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=26), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=298, start_offset=298, starts_line=True, line_number=25), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=8), + make_inst(opname='JUMP_FORWARD', arg=13, argval=120, argrepr='to L5', offset=92, start_offset=92, starts_line=False, line_number=8), + make_inst(opname='END_FOR', arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=True, line_number=3, label=4), + make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=96, start_offset=96, starts_line=False, line_number=3), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=98, start_offset=98, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=108, start_offset=108, starts_line=False, line_number=10), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=118, start_offset=118, starts_line=False, line_number=10), + make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=120, start_offset=120, starts_line=True, line_number=11, label=5), + make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=122, start_offset=122, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=40, argval=214, argrepr='to L8', offset=130, start_offset=130, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=134, start_offset=134, starts_line=False, line_number=11), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=136, start_offset=136, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=146, start_offset=146, starts_line=False, line_number=12), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=156, start_offset=156, starts_line=False, line_number=12), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=True, line_number=13), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=160, start_offset=160, starts_line=False, line_number=13), + make_inst(opname='BINARY_OP', arg=23, argval=23, argrepr='-=', offset=162, start_offset=162, starts_line=False, line_number=13, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=False, line_number=13), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=14), + make_inst(opname='LOAD_SMALL_INT', arg=6, argval=6, argrepr='', offset=178, start_offset=178, starts_line=False, line_number=14), + make_inst(opname='COMPARE_OP', arg=148, argval='>', argrepr='bool(>)', offset=180, start_offset=180, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=3, argval=194, argrepr='to L6', offset=184, start_offset=184, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=188, start_offset=188, starts_line=False, line_number=14), + make_inst(opname='JUMP_BACKWARD', arg=37, argval=120, argrepr='to L5', offset=190, start_offset=190, starts_line=True, line_number=15, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=194, start_offset=194, starts_line=True, line_number=16, label=6), + make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=196, start_offset=196, starts_line=False, line_number=16), + make_inst(opname='COMPARE_OP', arg=18, argval='<', argrepr='bool(<)', offset=198, start_offset=198, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=3, argval=212, argrepr='to L7', offset=202, start_offset=202, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=16), + make_inst(opname='JUMP_BACKWARD', arg=46, argval=120, argrepr='to L5', offset=208, start_offset=208, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='JUMP_FORWARD', arg=11, argval=236, argrepr='to L9', offset=212, start_offset=212, starts_line=True, line_number=17, label=7), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=214, start_offset=214, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=224, start_offset=224, starts_line=False, line_number=19), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=False, line_number=19), + make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=20, label=9), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=238, start_offset=238, starts_line=True, line_number=21), + make_inst(opname='LOAD_SMALL_INT', arg=0, argval=0, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21), + make_inst(opname='BINARY_OP', arg=11, argval=11, argrepr='/', offset=242, start_offset=242, starts_line=False, line_number=21, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=21), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=256, start_offset=256, starts_line=True, line_number=25), + make_inst(opname='COPY', arg=1, argval=1, argrepr='', offset=258, start_offset=258, starts_line=False, line_number=25), + make_inst(opname='LOAD_SPECIAL', arg=1, argval=1, argrepr='__exit__', offset=260, start_offset=260, starts_line=False, line_number=25), + make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25), + make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=264, start_offset=264, starts_line=False, line_number=25), + make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=266, start_offset=266, starts_line=False, line_number=25), + make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=276, start_offset=276, starts_line=False, line_number=25), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=278, start_offset=278, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=288, start_offset=288, starts_line=False, line_number=26), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=290, start_offset=290, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=26), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=True, line_number=25), make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=302, start_offset=302, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=314, start_offset=314, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=324, start_offset=324, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=28), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=336, start_offset=336, starts_line=False, line_number=28), - make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=28), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=True, line_number=25), - make_inst(opname='WITH_EXCEPT_START', arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25), - make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_JUMP_IF_TRUE', arg=2, argval=360, argrepr='to L11', offset=352, start_offset=352, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25), - make_inst(opname='RERAISE', arg=2, argval=2, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25, label=11), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=304, start_offset=304, starts_line=False, line_number=25), + make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=316, start_offset=316, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=326, start_offset=326, starts_line=False, line_number=28), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=336, start_offset=336, starts_line=False, line_number=28), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=338, start_offset=338, starts_line=False, line_number=28), + make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=28), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=True, line_number=25), + make_inst(opname='WITH_EXCEPT_START', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25), + make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=2, argval=362, argrepr='to L11', offset=354, start_offset=354, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25), + make_inst(opname='RERAISE', arg=2, argval=2, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25, label=11), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=25), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=25), - make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=29, argval=314, argrepr='to L10', offset=370, start_offset=370, starts_line=False, line_number=25), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=372, start_offset=372, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None), - make_inst(opname='LOAD_GLOBAL', arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=380, start_offset=380, starts_line=True, line_number=22, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='CHECK_EXC_MATCH', arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=22), - make_inst(opname='POP_JUMP_IF_FALSE', arg=15, argval=426, argrepr='to L12', offset=392, start_offset=392, starts_line=False, line_number=22, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=22), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=5, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=410, start_offset=410, starts_line=False, line_number=23), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=23), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23), - make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=314, argrepr='to L10', offset=424, start_offset=424, starts_line=False, line_number=23), - make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=426, start_offset=426, starts_line=True, line_number=22, label=12), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=436, start_offset=436, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=446, start_offset=446, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=456, start_offset=456, starts_line=False, line_number=28), - make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=460, start_offset=460, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=462, start_offset=462, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=False, line_number=25), + make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=29, argval=316, argrepr='to L10', offset=372, start_offset=372, starts_line=False, line_number=25), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=374, start_offset=374, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=None), + make_inst(opname='LOAD_GLOBAL', arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=382, start_offset=382, starts_line=True, line_number=22, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='CHECK_EXC_MATCH', arg=None, argval=None, argrepr='', offset=392, start_offset=392, starts_line=False, line_number=22), + make_inst(opname='POP_JUMP_IF_FALSE', arg=15, argval=428, argrepr='to L12', offset=394, start_offset=394, starts_line=False, line_number=22, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=22), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=402, start_offset=402, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=5, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=412, start_offset=412, starts_line=False, line_number=23), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=23), + make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=316, argrepr='to L10', offset=426, start_offset=426, starts_line=False, line_number=23), + make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=22, label=12), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=430, start_offset=430, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=436, start_offset=436, starts_line=False, line_number=None), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=438, start_offset=438, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=448, start_offset=448, starts_line=False, line_number=28), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=450, start_offset=450, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28), + make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=460, start_offset=460, starts_line=False, line_number=28), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=462, start_offset=462, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=466, start_offset=466, starts_line=False, line_number=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7a62219c139ee5..0d775b7bfa9f42 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1386,6 +1386,10 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[j] = next_stack; break; case GET_ITER: + next_stack = push_value(pop_value(next_stack), Iterator); + next_stack = push_value(next_stack, Iterator); + stacks[next_i] = next_stack; + break; case GET_AITER: next_stack = push_value(pop_value(next_stack), Iterator); stacks[next_i] = next_stack; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 8cedee31e08a00..bdd2744e266a89 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,6 +1,6 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { - 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, + 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,243,184,0,0,0,128,0,92,0,80,1, 71,0,114,0,92,0,80,1,71,1,114,1,91,2,32,0, 80,2,50,1,0,0,0,0,0,0,30,0,91,2,32,0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6592bc57ed20a1..b6970445384bc4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -389,7 +389,7 @@ dummy_func( } pure inst(POP_TOP, (value --)) { - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); } pure inst(PUSH_NULL, (-- res)) { @@ -405,9 +405,14 @@ dummy_func( PyStackRef_CLOSE(value); } - macro(POP_ITER) = POP_TOP; - no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { + inst(POP_ITER, (iter, index_or_null -- )) { + (void)index_or_null; + DEAD(index_or_null); + PyStackRef_CLOSE(iter); + } + + no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, index_or_null, value -- receiver, index_or_null)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { @@ -419,7 +424,9 @@ dummy_func( PyStackRef_CLOSE(value); } - tier1 inst(INSTRUMENTED_POP_ITER, (iter -- )) { + tier1 inst(INSTRUMENTED_POP_ITER, (iter, index_or_null -- )) { + (void)index_or_null; + DEAD(index_or_null); INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); PyStackRef_CLOSE(iter); } @@ -3019,7 +3026,7 @@ dummy_func( values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); } - inst(GET_ITER, (iterable -- iter)) { + inst(GET_ITER, (iterable -- iter, null)) { #ifdef Py_STATS _Py_GatherStats_GetIter(iterable); #endif @@ -3028,6 +3035,7 @@ dummy_func( PyStackRef_CLOSE(iterable); ERROR_IF(iter_o == NULL, error); iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { @@ -3074,7 +3082,7 @@ dummy_func( FOR_ITER_GEN, }; - specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { + specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter, null_or_index -- iter, null_or_index)) { #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -3086,7 +3094,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - replaced op(_FOR_ITER, (iter -- iter, next)) { + replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -3110,7 +3118,7 @@ dummy_func( // Common case: no jump, leave it to the code generator } - op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { + op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -3134,7 +3142,7 @@ dummy_func( macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; - inst(INSTRUMENTED_FOR_ITER, (unused/1, iter -- iter, next)) { + inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o != NULL) { @@ -3160,7 +3168,7 @@ dummy_func( } - op(_ITER_CHECK_LIST, (iter -- iter)) { + op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(iter_o) != &PyListIter_Type); #ifdef Py_GIL_DISABLED @@ -3171,7 +3179,7 @@ dummy_func( #endif } - replaced op(_ITER_JUMP_LIST, (iter -- iter)) { + replaced op(_ITER_JUMP_LIST, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(iter_o) == &PyListIter_Type); // For free-threaded Python, the loop exit can happen at any point during @@ -3199,7 +3207,7 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) { + op(_GUARD_NOT_EXHAUSTED_LIST, (iter, null_or_index -- iter, null_or_index)) { #ifndef Py_GIL_DISABLED PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; @@ -3213,7 +3221,7 @@ dummy_func( #endif } - replaced op(_ITER_NEXT_LIST, (iter -- iter, next)) { + replaced op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); @@ -3242,7 +3250,7 @@ dummy_func( } // Only used by Tier 2 - op(_ITER_NEXT_LIST_TIER_TWO, (iter -- iter, next)) { + op(_ITER_NEXT_LIST_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); @@ -3274,7 +3282,7 @@ dummy_func( _ITER_JUMP_LIST + _ITER_NEXT_LIST; - op(_ITER_CHECK_TUPLE, (iter -- iter)) { + op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(iter_o) != &PyTupleIter_Type); #ifdef Py_GIL_DISABLED @@ -3282,7 +3290,7 @@ dummy_func( #endif } - replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) { + replaced op(_ITER_JUMP_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); (void)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -3306,7 +3314,7 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter -- iter)) { + op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -3318,7 +3326,7 @@ dummy_func( EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); } - op(_ITER_NEXT_TUPLE, (iter -- iter, next)) { + op(_ITER_NEXT_TUPLE, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -3337,7 +3345,7 @@ dummy_func( _ITER_JUMP_TUPLE + _ITER_NEXT_TUPLE; - op(_ITER_CHECK_RANGE, (iter -- iter)) { + op(_ITER_CHECK_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3345,7 +3353,7 @@ dummy_func( #endif } - replaced op(_ITER_JUMP_RANGE, (iter -- iter)) { + replaced op(_ITER_JUMP_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3360,13 +3368,13 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) { + op(_GUARD_NOT_EXHAUSTED_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); EXIT_IF(r->len <= 0); } - op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3387,7 +3395,7 @@ dummy_func( _ITER_JUMP_RANGE + _ITER_NEXT_RANGE; - op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) { + op(_FOR_ITER_GEN_FRAME, (iter, null -- iter, null, gen_frame: _PyInterpreterFrame*)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); #ifdef Py_GIL_DISABLED diff --git a/Python/codegen.c b/Python/codegen.c index a7e412f7032c45..a5eeaae0cad6bd 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -525,6 +525,15 @@ codegen_unwind_fblock(compiler *c, location *ploc, return SUCCESS; case COMPILE_FBLOCK_FOR_LOOP: + /* Pop the iterator */ + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 3); + } + ADDOP(c, *ploc, POP_TOP); + ADDOP(c, *ploc, POP_TOP); + return SUCCESS; + + case COMPILE_FBLOCK_ASYNC_FOR_LOOP: /* Pop the iterator */ if (preserve_tos) { ADDOP_I(c, *ploc, SWAP, 2); @@ -629,7 +638,8 @@ codegen_unwind_fblock_stack(compiler *c, location *ploc, c, *ploc, "'break', 'continue' and 'return' cannot appear in an except* block"); } if (loop != NULL && (top->fb_type == COMPILE_FBLOCK_WHILE_LOOP || - top->fb_type == COMPILE_FBLOCK_FOR_LOOP)) { + top->fb_type == COMPILE_FBLOCK_FOR_LOOP || + top->fb_type == COMPILE_FBLOCK_ASYNC_FOR_LOOP)) { *loop = top; return SUCCESS; } @@ -2125,7 +2135,7 @@ codegen_async_for(compiler *c, stmt_ty s) ADDOP(c, LOC(s->v.AsyncFor.iter), GET_AITER); USE_LABEL(c, start); - RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FOR_LOOP, start, end, NULL)); + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_ASYNC_FOR_LOOP, start, end, NULL)); /* SETUP_FINALLY to guard the __anext__ call */ ADDOP_JUMP(c, loc, SETUP_FINALLY, except); @@ -2142,7 +2152,7 @@ codegen_async_for(compiler *c, stmt_ty s) /* Mark jump as artificial */ ADDOP_JUMP(c, NO_LOCATION, JUMP, start); - _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FOR_LOOP, start); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_ASYNC_FOR_LOOP, start); /* Except block for __anext__ */ USE_LABEL(c, except); @@ -3889,10 +3899,11 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) NEW_JUMP_TARGET_LABEL(c, loop); NEW_JUMP_TARGET_LABEL(c, cleanup); + ADDOP(c, loc, PUSH_NULL); // Push NULL index for loop USE_LABEL(c, loop); ADDOP_JUMP(c, loc, FOR_ITER, cleanup); if (const_oparg == CONSTANT_BUILTIN_TUPLE) { - ADDOP_I(c, loc, LIST_APPEND, 2); + ADDOP_I(c, loc, LIST_APPEND, 3); ADDOP_JUMP(c, loc, JUMP, loop); } else { @@ -4366,13 +4377,12 @@ codegen_sync_comprehension_generator(compiler *c, location loc, } if (IS_JUMP_TARGET_LABEL(start)) { VISIT(c, expr, gen->iter); - ADDOP(c, LOC(gen->iter), GET_ITER); } } } if (IS_JUMP_TARGET_LABEL(start)) { - depth++; + depth += 2; ADDOP(c, LOC(gen->iter), GET_ITER); USE_LABEL(c, start); ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor); @@ -4681,19 +4691,6 @@ pop_inlined_comprehension_state(compiler *c, location loc, return SUCCESS; } -static inline int -codegen_comprehension_iter(compiler *c, comprehension_ty comp) -{ - VISIT(c, expr, comp->iter); - if (comp->is_async) { - ADDOP(c, LOC(comp->iter), GET_AITER); - } - else { - ADDOP(c, LOC(comp->iter), GET_ITER); - } - return SUCCESS; -} - static int codegen_comprehension(compiler *c, expr_ty e, int type, identifier name, asdl_comprehension_seq *generators, expr_ty elt, @@ -4713,9 +4710,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, outermost = (comprehension_ty) asdl_seq_GET(generators, 0); if (is_inlined) { - if (codegen_comprehension_iter(c, outermost)) { - goto error; - } + VISIT(c, expr, outermost->iter); if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { goto error; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4e57906c84a154..c97910f07eb847 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -584,7 +584,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -609,6 +609,20 @@ break; } + case _POP_ITER: { + _PyStackRef index_or_null; + _PyStackRef iter; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } + case _END_SEND: { _PyStackRef value; _PyStackRef receiver; @@ -4074,6 +4088,7 @@ case _GET_ITER: { _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4092,8 +4107,10 @@ JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4141,7 +4158,7 @@ case _FOR_ITER_TIER_TWO: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -4175,7 +4192,7 @@ case _ITER_CHECK_LIST: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyListIter_Type) { UOP_STAT_INC(uopcode, miss); @@ -4200,7 +4217,7 @@ case _GUARD_NOT_EXHAUSTED_LIST: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; #ifndef Py_GIL_DISABLED PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; @@ -4226,7 +4243,7 @@ case _ITER_NEXT_LIST_TIER_TWO: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyListIterObject *it = (_PyListIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyListIter_Type); @@ -4264,7 +4281,7 @@ case _ITER_CHECK_TUPLE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTupleIter_Type) { UOP_STAT_INC(uopcode, miss); @@ -4283,7 +4300,7 @@ case _GUARD_NOT_EXHAUSTED_TUPLE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -4305,7 +4322,7 @@ case _ITER_NEXT_TUPLE: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; assert(Py_TYPE(iter_o) == &PyTupleIter_Type); @@ -4324,7 +4341,7 @@ case _ITER_CHECK_RANGE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); @@ -4343,7 +4360,7 @@ case _GUARD_NOT_EXHAUSTED_RANGE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { @@ -4356,7 +4373,7 @@ case _ITER_NEXT_RANGE: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -4381,7 +4398,7 @@ _PyStackRef iter; _PyInterpreterFrame *gen_frame; oparg = CURRENT_OPARG(); - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/flowgraph.c b/Python/flowgraph.c index e0bb5615db3e0b..04b2ae839def9f 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -273,7 +273,7 @@ basicblock_insert_instruction(basicblock *block, int pos, cfg_instr *instr) { } /* For debugging purposes only */ -#if 0 +#if 1 static void dump_instr(cfg_instr *i) { @@ -299,29 +299,43 @@ basicblock_returns(const basicblock *b) { } static void -dump_basicblock(const basicblock *b) +dump_basicblock(const basicblock *b, bool highlight) { const char *b_return = basicblock_returns(b) ? "return " : ""; + if (highlight) { + fprintf(stderr, ">>> "); + } fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, preds: %d %s\n", b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, b->b_startdepth, b->b_predecessors, b_return); + int depth = b->b_startdepth; if (b->b_instr) { int i; for (i = 0; i < b->b_iused; i++) { - fprintf(stderr, " [%02d] ", i); + fprintf(stderr, " [%02d] depth: %d ", i, depth); dump_instr(b->b_instr + i); + + int popped = _PyOpcode_num_popped(b->b_instr[i].i_opcode, b->b_instr[i].i_oparg); + int pushed = _PyOpcode_num_pushed(b->b_instr[i].i_opcode, b->b_instr[i].i_oparg); + depth += (pushed - popped); } } } void -_PyCfgBuilder_DumpGraph(const basicblock *entryblock) +_PyCfgBuilder_DumpGraph(const basicblock *entryblock, const basicblock *mark) { for (const basicblock *b = entryblock; b != NULL; b = b->b_next) { - dump_basicblock(b); + dump_basicblock(b, b == mark); } } +#define DUMP(B, M) _PyCfgBuilder_DumpGraph((B), (M)) + +#else + +#define DUMP(B, M) ((void)0) + #endif @@ -821,6 +835,7 @@ calculate_stackdepth(cfg_builder *g) int maxdepth = 0; basicblock **sp = stack; if (stackdepth_push(&sp, entryblock, 0) < 0) { + DUMP(entryblock, entryblock); goto error; } while (sp != stack) { @@ -841,6 +856,7 @@ calculate_stackdepth(cfg_builder *g) if (new_depth < 0) { PyErr_Format(PyExc_ValueError, "Invalid CFG, stack underflow"); + DUMP(entryblock, b); goto error; } maxdepth = Py_MAX(maxdepth, depth); @@ -855,6 +871,7 @@ calculate_stackdepth(cfg_builder *g) assert(target_depth >= 0); /* invalid code or bug in stackdepth() */ maxdepth = Py_MAX(maxdepth, depth); if (stackdepth_push(&sp, instr->i_target, target_depth) < 0) { + DUMP(entryblock, instr->i_target); goto error; } } @@ -871,6 +888,7 @@ calculate_stackdepth(cfg_builder *g) if (next != NULL) { assert(BB_HAS_FALLTHROUGH(b)); if (stackdepth_push(&sp, next, depth) < 0) { + DUMP(entryblock, next); goto error; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7d3e6c7cbc9eab..1a26aa91482323 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5512,7 +5512,7 @@ _PyStackRef next; // _SPECIALIZE_FOR_ITER { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT @@ -5584,7 +5584,7 @@ } // _FOR_ITER_GEN_FRAME { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5645,7 +5645,7 @@ /* Skip 1 cache entry */ // _ITER_CHECK_LIST { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyListIter_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5744,7 +5744,7 @@ /* Skip 1 cache entry */ // _ITER_CHECK_RANGE { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5811,7 +5811,7 @@ /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTupleIter_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5994,6 +5994,7 @@ INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6012,8 +6013,10 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6777,7 +6780,7 @@ _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -6841,7 +6844,7 @@ _PyStackRef iter; _PyStackRef next; /* Skip 1 cache entry */ - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -7134,9 +7137,12 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; - iter = stack_pointer[-1]; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -1; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); @@ -9942,12 +9948,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -10095,7 +10104,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 679240b6efa315..2f54a240950b5f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -131,6 +131,12 @@ break; } + case _POP_ITER: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _END_SEND: { JitOptSymbol *val; val = sym_new_not_null(ctx); @@ -1484,8 +1490,13 @@ case _GET_ITER: { JitOptSymbol *iter; + JitOptSymbol *null; iter = sym_new_not_null(ctx); + null = sym_new_null(ctx); stack_pointer[-1] = iter; + stack_pointer[0] = null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } From 38a429f679839cb2e548a198c708064e9793ba82 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 11 Apr 2025 18:07:29 +0100 Subject: [PATCH 02/21] use tagged ints for indexes. Work in progress --- Include/internal/pycore_ceval.h | 2 + Include/internal/pycore_code.h | 2 +- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_stackref.h | 18 +- Include/internal/pycore_uop_ids.h | 7 +- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 253 ++++++++++---------- Python/ceval.c | 18 ++ Python/codegen.c | 2 +- Python/executor_cases.c.h | 152 ++++++------ Python/generated_cases.c.h | 257 +++++++++++---------- Python/optimizer_cases.c.h | 6 +- Python/specialize.c | 57 +++-- Python/stackrefs.c | 8 + Tools/cases_generator/analyzer.py | 5 + Tools/cases_generator/generators_common.py | 1 + 16 files changed, 420 insertions(+), 372 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 115dd40b8bdfce..63784da16cb8c7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -377,6 +377,8 @@ PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyS extern int _PyRunRemoteDebugger(PyThreadState *tstate); #endif +_PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 635d2b24f4bdff..c11d0e104a9094 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -313,7 +313,7 @@ extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 44bd6de9da6740..34060b7133f24c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1150,7 +1150,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index a2acf311ff4c65..dbd7a061ad3771 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -233,6 +233,8 @@ extern intptr_t PyStackRef_UntagInt(_PyStackRef ref); extern _PyStackRef PyStackRef_TagInt(intptr_t i); +extern _PyStackRef PyStackRef_IncrementTaggedInt(_PyStackRef ref); + extern bool PyStackRef_IsNullOrInt(_PyStackRef ref); @@ -262,6 +264,14 @@ PyStackRef_UntagInt(_PyStackRef i) } +static inline _PyStackRef +PyStackRef_IncrementTaggedInt(_PyStackRef ref) +{ + assert(ref.bits != (uintptr_t)-1); // Overflow + return (_PyStackRef){ .bits = ref.bits + 4 }; +} + + #ifdef Py_GIL_DISABLED #define Py_TAG_DEFERRED (1) @@ -686,7 +696,13 @@ PyStackRef_XCLOSE(_PyStackRef ref) #endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) -#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) +static inline PyTypeObject * +PyStackRef_TYPE(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return &PyLong_Type; + } + return Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)); +} // Converts a PyStackRef back to a PyObject *, converting the // stackref to a new reference. diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3155b299f3db4b..754f43cb5aaca1 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -256,14 +256,9 @@ extern "C" { #define _MONITOR_RESUME 471 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -<<<<<<< HEAD +#define _POP_ITER POP_ITER #define _POP_JUMP_IF_FALSE 472 #define _POP_JUMP_IF_TRUE 473 -======= -#define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 465 -#define _POP_JUMP_IF_TRUE 466 ->>>>>>> d908cf7e4c1 (FOR_ITER now pushes NULL as well as the iterator. Preparation for virtual iterators) #define _POP_TOP POP_TOP #define _POP_TOP_LOAD_CONST_INLINE 474 #define _POP_TOP_LOAD_CONST_INLINE_BORROW 475 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index c339888b892f5b..798f7c6bce9a75 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -202,7 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, - [_ITER_NEXT_LIST_TIER_TWO] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_ITER_NEXT_LIST_TIER_TWO] = HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b6970445384bc4..deefe5c9b7b46e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3026,16 +3026,24 @@ dummy_func( values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); } - inst(GET_ITER, (iterable -- iter, null)) { + inst(GET_ITER, (iterable -- iter, index_or_null)) { #ifdef Py_STATS _Py_GatherStats_GetIter(iterable); #endif /* before: [obj]; after [getiter(obj)] */ - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - PyStackRef_CLOSE(iterable); - ERROR_IF(iter_o == NULL, error); - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + DEAD(iterable); + index_or_null = PyStackRef_TagInt(0); + } + else { + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + PyStackRef_CLOSE(iterable); + ERROR_IF(iter_o == NULL, error); + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + } } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { @@ -3086,7 +3094,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -3097,30 +3105,41 @@ dummy_func( replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); + if (PyStackRef_IsTaggedInt(null_or_index)) { + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + if (PyStackRef_IsNull(next)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + DISPATCH(); + } + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + } + else { + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { + ERROR_NO_POP(); + } + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); } - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); + next = PyStackRef_FromPyObjectSteal(next_o); } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator } op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + EXIT_IF(!PyStackRef_IsNull(null_or_index)); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { @@ -3144,61 +3163,62 @@ dummy_func( inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); + if (PyStackRef_IsTaggedInt(null_or_index)) { + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + if (PyStackRef_IsNull(next)) { + JUMPBY(oparg + 1); + DISPATCH(); + } INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o != NULL) { + next = PyStackRef_FromPyObjectSteal(next_o); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } + else { + if (_PyErr_Occurred(tstate)) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + if (!matches) { + ERROR_NO_POP(); + } + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); } - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + /* Skip END_FOR */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Skip END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); } } op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - EXIT_IF(Py_TYPE(iter_o) != &PyListIter_Type); + EXIT_IF(Py_TYPE(iter_o) != &PyList_Type); + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - EXIT_IF(!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)); + EXIT_IF(!_Py_IsOwnedByCurrentThread(iter_o) || + !_PyObject_GC_IS_SHARED(iter_o)); #endif } replaced op(_ITER_JUMP_LIST, (iter, null_or_index -- iter, null_or_index)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(iter_o) == &PyListIter_Type); -// For free-threaded Python, the loop exit can happen at any point during -// item retrieval, so it doesn't make much sense to check and jump -// separately before item retrieval. Any length check we do here can be -// invalid by the time we actually try to fetch the item. #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - (void)iter_o; + // For free-threaded Python, the loop exit can happen at any point during + // item retrieval, so it doesn't make much sense to check and jump + // separately before item retrieval. Any length check we do here can be + // invalid by the time we actually try to fetch the item. #else - _PyListIterObject *it = (_PyListIterObject *)iter_o; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3209,71 +3229,61 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_LIST, (iter, null_or_index -- iter, null_or_index)) { #ifndef Py_GIL_DISABLED - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - EXIT_IF(seq == NULL); - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - EXIT_IF(1); - } + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + EXIT_IF((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)); #endif } replaced op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); // A negative result means we lost a race with another thread // and we need to take the slow path. DEOPT_IF(result < 0); if (result == 0) { - it->it_index = -1; + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + intptr_t i = PyStackRef_UntagInt(null_or_index); + assert(i < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, i)); #endif + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + assert(PyStackRef_UntagInt(null_or_index) == i + 1); } // Only used by Tier 2 op(_ITER_NEXT_LIST_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); // A negative result means we lost a race with another thread // and we need to take the slow path. - EXIT_IF(result < 0); + DEOPT_IF(result < 0); if (result == 0) { - it->it_index = -1; - EXIT_IF(1); + null_or_index = PyStackRef_TagInt(-1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); } macro(FOR_ITER_LIST) = @@ -3284,29 +3294,20 @@ dummy_func( op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - EXIT_IF(Py_TYPE(iter_o) != &PyTupleIter_Type); + EXIT_IF(Py_TYPE(iter_o) != &PyTuple_Type); + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); #endif } replaced op(_ITER_JUMP_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - (void)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { -#ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } -#endif + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3315,28 +3316,18 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - PyTupleObject *seq = it->it_seq; - EXIT_IF(seq == NULL); - EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + EXIT_IF((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)); } op(_ITER_NEXT_TUPLE, (iter, null_or_index -- iter, null_or_index, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); } macro(FOR_ITER_TUPLE) = diff --git a/Python/ceval.c b/Python/ceval.c index 19a1c9529dd9aa..9e2204378cf1f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -155,6 +155,9 @@ dump_item(_PyStackRef item) printf(""); return; } + if (PyList_CheckExact(obj)) { + printf("len=%ld ", Py_SIZE(obj)); + } // Don't call __repr__(), it might recurse into the interpreter. printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); } @@ -271,6 +274,7 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } } + // lltrace = 5; if (lltrace >= 5) { lltrace_resume_frame(frame); } @@ -3421,6 +3425,20 @@ _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *na return value; } +_PyStackRef +_PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index) +{ + assert(PyStackRef_IsTaggedInt(index)); + assert(Py_TYPE(seq) == &PyTuple_Type || Py_TYPE(seq) == &PyList_Type); + size_t size = Py_SIZE(seq); + intptr_t i = PyStackRef_UntagInt(index); + if ((size_t)i >= size) { + return PyStackRef_NULL; + } + PyObject *next_o = PySequence_Fast_GET_ITEM(seq, i); + return PyStackRef_FromPyObjectNew(next_o); +} + /* Check if a 'cls' provides the given special method. */ static inline int type_has_special_method(PyTypeObject *cls, PyObject *name) diff --git a/Python/codegen.c b/Python/codegen.c index a5eeaae0cad6bd..9e662552ccca14 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4477,9 +4477,9 @@ codegen_async_comprehension_generator(compiler *c, location loc, else { /* Sub-iter - calculate on the fly */ VISIT(c, expr, gen->iter); - ADDOP(c, LOC(gen->iter), GET_AITER); } } + ADDOP(c, LOC(gen->iter), GET_AITER); USE_LABEL(c, start); /* Runtime will push a block here, so we need to account for that */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c97910f07eb847..aa179bdd60f548 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4088,29 +4088,38 @@ case _GET_ITER: { _PyStackRef iterable; _PyStackRef iter; - _PyStackRef null; + _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_ERROR(); + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - stack_pointer[0] = iter; - stack_pointer[1] = null; - stack_pointer += 2; + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4156,10 +4165,16 @@ /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ case _FOR_ITER_TIER_TWO: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyStackRef_IsNull(null_or_index)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4191,21 +4206,19 @@ /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ case _ITER_CHECK_LIST: { + _PyStackRef null_or_index; _PyStackRef iter; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyListIter_Type) { + if (Py_TYPE(iter_o) != &PyList_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) || + !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4216,24 +4229,17 @@ /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_LIST: { + _PyStackRef null_or_index; _PyStackRef iter; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; #ifndef Py_GIL_DISABLED - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - if (seq == NULL) { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } #endif break; } @@ -4241,38 +4247,37 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ case _ITER_NEXT_LIST_TIER_TWO: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; + oparg = CURRENT_OPARG(); + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); if (result < 0) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } if (result == 0) { - it->it_index = -1; - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4280,13 +4285,16 @@ } case _ITER_CHECK_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTupleIter_Type) { + if (Py_TYPE(iter_o) != &PyTuple_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced(iter_o)) { UOP_STAT_INC(uopcode, miss); @@ -4299,20 +4307,13 @@ /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - PyTupleObject *seq = it->it_seq; - if (seq == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (it->it_index >= PyTuple_GET_SIZE(seq)) { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4320,19 +4321,18 @@ } case _ITER_NEXT_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1a26aa91482323..1b7327c80b731e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5509,9 +5509,11 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; // _SPECIALIZE_FOR_ITER { + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; @@ -5519,7 +5521,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -5530,29 +5532,44 @@ // _FOR_ITER { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); + if (PyStackRef_IsTaggedInt(null_or_index)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(next)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + JUMPBY(oparg + 1); + DISPATCH(); } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - JUMPBY(oparg + 1); - DISPATCH(); + next = PyStackRef_FromPyObjectSteal(next_o); } - next = PyStackRef_FromPyObjectSteal(next_o); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5641,26 +5658,23 @@ INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST { + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyListIter_Type) { + if (Py_TYPE(iter_o) != &PyList_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) || + !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); @@ -5669,42 +5683,30 @@ } // _ITER_JUMP_LIST { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(iter_o) == &PyListIter_Type); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - (void)iter_o; + #else - _PyListIterObject *it = (_PyListIterObject *)iter_o; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } #endif } // _ITER_NEXT_LIST { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); @@ -5712,16 +5714,20 @@ JUMP_TO_PREDICTED(FOR_ITER); } if (result == 0) { - it->it_index = -1; + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + intptr_t i = PyStackRef_UntagInt(null_or_index); + assert(i < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, i)); #endif + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + assert(PyStackRef_UntagInt(null_or_index) == i + 1); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5807,17 +5813,20 @@ INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE { + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTupleIter_Type) { + if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); @@ -5828,42 +5837,27 @@ } // _ITER_JUMP_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - (void)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif - + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } } // _ITER_NEXT_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5994,29 +5988,38 @@ INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; - _PyStackRef null; + _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - stack_pointer[0] = iter; - stack_pointer[1] = null; - stack_pointer += 2; + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6842,34 +6845,48 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ + null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); + if (PyStackRef_IsTaggedInt(null_or_index)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + next = _PyForIter_NextWithIndex(iter_o, null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(next)) { + JUMPBY(oparg + 1); + DISPATCH(); + } INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o != NULL) { + next = PyStackRef_FromPyObjectSteal(next_o); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + } + else { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + JUMPBY(oparg + 1); + DISPATCH(); } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - JUMPBY(oparg + 1); - DISPATCH(); } stack_pointer[0] = next; stack_pointer += 1; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2f54a240950b5f..1bf59c8c0bec80 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1490,11 +1490,11 @@ case _GET_ITER: { JitOptSymbol *iter; - JitOptSymbol *null; + JitOptSymbol *index_or_null; iter = sym_new_not_null(ctx); - null = sym_new_null(ctx); + index_or_null = sym_new_not_null(ctx); stack_pointer[-1] = iter; - stack_pointer[0] = null; + stack_pointer[0] = index_or_null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Python/specialize.c b/Python/specialize.c index 562e4a19d64a7f..bcaad21bfddc26 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2898,7 +2898,7 @@ int #endif // Py_STATS Py_NO_INLINE void -_Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg) { assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); @@ -2913,38 +2913,33 @@ _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) if (!_PyObject_IsUniquelyReferenced(iter_o)) goto failure; #endif - if (tp == &PyListIter_Type) { -#ifdef Py_GIL_DISABLED - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) && - !_PyObject_GC_IS_SHARED(it->it_seq)) { - // Maybe this should just set GC_IS_SHARED in a critical - // section, instead of leaving it to the first iteration? - goto failure; + if (PyStackRef_IsNull(null_or_index)) { + if (tp == &PyRangeIter_Type) { + specialize(instr, FOR_ITER_RANGE); + return; + } + else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { + // Generators are very much not thread-safe, so don't worry about + // the specialization not being thread-safe. + assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || + instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR + ); + /* Don't specialize if PEP 523 is active */ + if (_PyInterpreterState_GET()->eval_frame) + goto failure; + specialize(instr, FOR_ITER_GEN); + return; } -#endif - specialize(instr, FOR_ITER_LIST); - return; - } - else if (tp == &PyTupleIter_Type) { - specialize(instr, FOR_ITER_TUPLE); - return; - } - else if (tp == &PyRangeIter_Type) { - specialize(instr, FOR_ITER_RANGE); - return; } - else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { - // Generators are very much not thread-safe, so don't worry about - // the specialization not being thread-safe. - assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || - instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR - ); - /* Don't specialize if PEP 523 is active */ - if (_PyInterpreterState_GET()->eval_frame) - goto failure; - specialize(instr, FOR_ITER_GEN); - return; + else { + if (tp == &PyList_Type) { + specialize(instr, FOR_ITER_LIST); + return; + } + else if (tp == &PyTuple_Type) { + specialize(instr, FOR_ITER_TUPLE); + return; + } } failure: SPECIALIZATION_FAIL(FOR_ITER, diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 979a6b1c62820a..c64525b21d35fe 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -216,4 +216,12 @@ PyStackRef_IsNullOrInt(_PyStackRef ref) return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); } +_PyStackRef +PyStackRef_IncrementTaggedInt(_PyStackRef ref) +{ + assert(ref.index != (uintptr_t)-1); // Overflow + return (_PyStackRef){ .index = ref.index + 2 }; +} + + #endif diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index dddbf2cf872e3d..8463ca4f979126 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -602,6 +602,10 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_None", "PyStackRef_TYPE", "PyStackRef_True", + "PyStackRef_TagInt", + "PyStackRef_UntagInt", + "PyStackRef_IsTaggedInt", + "PyStackRef_IncrementTaggedInt", "PyTuple_GET_ITEM", "PyTuple_GET_SIZE", "PyType_HasFeature", @@ -683,6 +687,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_UntagInt", ) + def check_escaping_calls(instr: parser.CodeDef, escapes: dict[SimpleStmt, EscapingCall]) -> None: error: lexer.Token | None = None calls = {e.call for e in escapes.values()} diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 9ba0767cba35a0..025dd4dd3b2910 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -140,6 +140,7 @@ def dispatch( ) -> bool: if storage.spilled: raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) self.emit(tkn) return False From 2caf56f7384ed9909464d6c0c339594fbb611551 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 15 Apr 2025 16:50:57 +0100 Subject: [PATCH 03/21] Remove debug code --- Python/bytecodes.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index deefe5c9b7b46e..72cdfa47838fc0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3253,12 +3253,9 @@ dummy_func( DISPATCH(); } #else - intptr_t i = PyStackRef_UntagInt(null_or_index); - assert(i < PyList_GET_SIZE(list_o)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, i)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); - assert(PyStackRef_UntagInt(null_or_index) == i + 1); } // Only used by Tier 2 From 588943a2cd1e24154a379901de8381b3626c8fc4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 15 Apr 2025 18:07:06 +0100 Subject: [PATCH 04/21] Regenerate files --- Python/generated_cases.c.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1b7327c80b731e..7ee23f0a7fd5f0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5720,12 +5720,9 @@ DISPATCH(); } #else - intptr_t i = PyStackRef_UntagInt(null_or_index); - assert(i < PyList_GET_SIZE(list_o)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, i)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); - assert(PyStackRef_UntagInt(null_or_index) == i + 1); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; From aa62e3976b843c8ef4adf987509e86a0337c4eed Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 16 Apr 2025 09:37:36 +0100 Subject: [PATCH 05/21] Tidy up list of non-escaping functions --- Tools/cases_generator/analyzer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 8463ca4f979126..c97d96c9e7f7b1 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -602,10 +602,6 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_None", "PyStackRef_TYPE", "PyStackRef_True", - "PyStackRef_TagInt", - "PyStackRef_UntagInt", - "PyStackRef_IsTaggedInt", - "PyStackRef_IncrementTaggedInt", "PyTuple_GET_ITEM", "PyTuple_GET_SIZE", "PyType_HasFeature", @@ -685,6 +681,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_IsTaggedInt", "PyStackRef_TagInt", "PyStackRef_UntagInt", + "PyStackRef_IncrementTaggedInt", + "PyStackRef_IsNullOrInt", ) From 88562ec76b46e50b4202d89f99efdf331529082a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 16 Apr 2025 09:58:22 +0100 Subject: [PATCH 06/21] Fix up list iteration for FT build --- Python/bytecodes.c | 3 +-- Python/executor_cases.c.h | 3 +-- Python/generated_cases.c.h | 3 +-- Python/specialize.c | 9 +++------ 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 72cdfa47838fc0..05ff8e62a87b1f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3202,8 +3202,7 @@ dummy_func( EXIT_IF(Py_TYPE(iter_o) != &PyList_Type); assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - EXIT_IF(!_Py_IsOwnedByCurrentThread(iter_o) || - !_PyObject_GC_IS_SHARED(iter_o)); + EXIT_IF(!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)); #endif } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index aa179bdd60f548..11775e612e6975 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4217,8 +4217,7 @@ } assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) || - !_PyObject_GC_IS_SHARED(iter_o)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7ee23f0a7fd5f0..998e981662448e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5673,8 +5673,7 @@ } assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) || - !_PyObject_GC_IS_SHARED(iter_o)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); diff --git a/Python/specialize.c b/Python/specialize.c index bcaad21bfddc26..3e1fe5a7070fdc 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2905,13 +2905,10 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyTypeObject *tp = Py_TYPE(iter_o); #ifdef Py_GIL_DISABLED - // Only specialize for uniquely referenced iterators, so that we know - // they're only referenced by this one thread. This is more limiting - // than we need (even `it = iter(mylist); for item in it:` won't get - // specialized) but we don't have a way to check whether we're the only - // _thread_ who has access to the object. - if (!_PyObject_IsUniquelyReferenced(iter_o)) + // Only specialize for lists owned by this thread or shared + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { goto failure; + } #endif if (PyStackRef_IsNull(null_or_index)) { if (tp == &PyRangeIter_Type) { From 025049dab078f288c1ee75e9ee0775bbf320c03b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 16 Apr 2025 10:47:00 +0100 Subject: [PATCH 07/21] Fix tier 2 FT build --- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 8 +------- Python/executor_cases.c.h | 9 +-------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 798f7c6bce9a75..ea47602dea37e7 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -202,7 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, - [_ITER_NEXT_LIST_TIER_TWO] = HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 05ff8e62a87b1f..da45b9aa66e1db 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3268,13 +3268,7 @@ dummy_func( int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); // A negative result means we lost a race with another thread // and we need to take the slow path. - DEOPT_IF(result < 0); - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } + DEOPT_IF(result <= 0); #else assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 11775e612e6975..1b1ab6efdb6a72 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4249,7 +4249,6 @@ _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; - oparg = CURRENT_OPARG(); null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); @@ -4261,16 +4260,10 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); - if (result < 0) { + if (result <= 0) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - DISPATCH(); - } #else assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); From 35e389ddf5f918da615f0979357c9b94aaa29bbb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 10:11:19 +0100 Subject: [PATCH 08/21] Remove debugging code --- Python/ceval.c | 4 ---- Python/flowgraph.c | 12 +----------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 9e2204378cf1f8..dfe8abbbc70164 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -155,9 +155,6 @@ dump_item(_PyStackRef item) printf(""); return; } - if (PyList_CheckExact(obj)) { - printf("len=%ld ", Py_SIZE(obj)); - } // Don't call __repr__(), it might recurse into the interpreter. printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); } @@ -274,7 +271,6 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } } - // lltrace = 5; if (lltrace >= 5) { lltrace_resume_frame(frame); } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 04b2ae839def9f..c3ac1513cc130b 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -273,7 +273,7 @@ basicblock_insert_instruction(basicblock *block, int pos, cfg_instr *instr) { } /* For debugging purposes only */ -#if 1 +#if 0 static void dump_instr(cfg_instr *i) { @@ -330,12 +330,6 @@ _PyCfgBuilder_DumpGraph(const basicblock *entryblock, const basicblock *mark) } } -#define DUMP(B, M) _PyCfgBuilder_DumpGraph((B), (M)) - -#else - -#define DUMP(B, M) ((void)0) - #endif @@ -835,7 +829,6 @@ calculate_stackdepth(cfg_builder *g) int maxdepth = 0; basicblock **sp = stack; if (stackdepth_push(&sp, entryblock, 0) < 0) { - DUMP(entryblock, entryblock); goto error; } while (sp != stack) { @@ -856,7 +849,6 @@ calculate_stackdepth(cfg_builder *g) if (new_depth < 0) { PyErr_Format(PyExc_ValueError, "Invalid CFG, stack underflow"); - DUMP(entryblock, b); goto error; } maxdepth = Py_MAX(maxdepth, depth); @@ -871,7 +863,6 @@ calculate_stackdepth(cfg_builder *g) assert(target_depth >= 0); /* invalid code or bug in stackdepth() */ maxdepth = Py_MAX(maxdepth, depth); if (stackdepth_push(&sp, instr->i_target, target_depth) < 0) { - DUMP(entryblock, instr->i_target); goto error; } } @@ -888,7 +879,6 @@ calculate_stackdepth(cfg_builder *g) if (next != NULL) { assert(BB_HAS_FALLTHROUGH(b)); if (stackdepth_push(&sp, next, depth) < 0) { - DUMP(entryblock, next); goto error; } } From ed89950013bd93a8fd363a3b2b346cf40e7f4389 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 10:19:42 +0100 Subject: [PATCH 09/21] Rename function --- Include/internal/pycore_stackref.h | 7 ++++--- Python/bytecodes.c | 8 ++++---- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 6 +++--- Python/stackrefs.c | 2 +- Tools/cases_generator/analyzer.py | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index dbd7a061ad3771..ad3699fef1d58c 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -233,7 +233,8 @@ extern intptr_t PyStackRef_UntagInt(_PyStackRef ref); extern _PyStackRef PyStackRef_TagInt(intptr_t i); -extern _PyStackRef PyStackRef_IncrementTaggedInt(_PyStackRef ref); +/* Increments a tagged int, but does not check for overflow */ +extern _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref); extern bool PyStackRef_IsNullOrInt(_PyStackRef ref); @@ -265,9 +266,9 @@ PyStackRef_UntagInt(_PyStackRef i) static inline _PyStackRef -PyStackRef_IncrementTaggedInt(_PyStackRef ref) +PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) { - assert(ref.bits != (uintptr_t)-1); // Overflow + assert(ref.bits != (uintptr_t)-1); // Deosn't overflow return (_PyStackRef){ .bits = ref.bits + 4 }; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8a5056dcd35ed9..d42a1c4b703485 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3112,7 +3112,7 @@ dummy_func( JUMPBY(oparg + 1); DISPATCH(); } - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } else { PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); @@ -3254,7 +3254,7 @@ dummy_func( #else next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } // Only used by Tier 2 @@ -3273,7 +3273,7 @@ dummy_func( assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } macro(FOR_ITER_LIST) = @@ -3317,7 +3317,7 @@ dummy_func( uintptr_t i = PyStackRef_UntagInt(null_or_index); assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } macro(FOR_ITER_TUPLE) = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1b1ab6efdb6a72..3c9544bf266ec3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4268,7 +4268,7 @@ assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; @@ -4323,7 +4323,7 @@ uintptr_t i = PyStackRef_UntagInt(null_or_index); assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e9d8266e2ad11c..eed7c908f75585 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5542,7 +5542,7 @@ stack_pointer[-1] = null_or_index; DISPATCH(); } - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } else { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5721,7 +5721,7 @@ #else next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; @@ -5851,7 +5851,7 @@ uintptr_t i = PyStackRef_UntagInt(null_or_index); assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedInt(null_or_index); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; diff --git a/Python/stackrefs.c b/Python/stackrefs.c index c64525b21d35fe..7671e40cfeb8e5 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -217,7 +217,7 @@ PyStackRef_IsNullOrInt(_PyStackRef ref) } _PyStackRef -PyStackRef_IncrementTaggedInt(_PyStackRef ref) +PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) { assert(ref.index != (uintptr_t)-1); // Overflow return (_PyStackRef){ .index = ref.index + 2 }; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index ddc1d8d7a5ee5b..f0455e923e7b2a 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -679,7 +679,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_IsTaggedInt", "PyStackRef_TagInt", "PyStackRef_UntagInt", - "PyStackRef_IncrementTaggedInt", + "PyStackRef_IncrementTaggedIntNoOverflow", "PyStackRef_IsNullOrInt", ) From ec8d7970c69fa36b19ff20cb38a1ee015ca9b75d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 11:32:00 +0100 Subject: [PATCH 10/21] Restrict specialization in free-threaded build --- Python/bytecodes.c | 3 --- Python/executor_cases.c.h | 6 ------ Python/generated_cases.c.h | 7 ------- Python/specialize.c | 21 ++++++++++++++++----- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d42a1c4b703485..f4474ac90f79e8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3286,9 +3286,6 @@ dummy_func( PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(iter_o) != &PyTuple_Type); assert(PyStackRef_IsTaggedInt(null_or_index)); -#ifdef Py_GIL_DISABLED - EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); -#endif } replaced op(_ITER_JUMP_TUPLE, (iter, null_or_index -- iter, null_or_index)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3c9544bf266ec3..769bc2c84bb621 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4287,12 +4287,6 @@ JUMP_TO_JUMP_TARGET(); } assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - #endif break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index eed7c908f75585..b1d74a9303c4fe 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5823,13 +5823,6 @@ JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - #endif } // _ITER_JUMP_TUPLE { diff --git a/Python/specialize.c b/Python/specialize.c index 3e1fe5a7070fdc..1da692ebe065ef 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2904,13 +2904,18 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyTypeObject *tp = Py_TYPE(iter_o); + + if (PyStackRef_IsNull(null_or_index)) { #ifdef Py_GIL_DISABLED - // Only specialize for lists owned by this thread or shared - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - goto failure; - } + // Only specialize for uniquely referenced iterators, so that we know + // they're only referenced by this one thread. This is more limiting + // than we need (even `it = iter(mylist); for item in it:` won't get + // specialized) but we don't have a way to check whether we're the only + // _thread_ who has access to the object. + if (!_PyObject_IsUniquelyReferenced(iter_o)) { + goto failure; + } #endif - if (PyStackRef_IsNull(null_or_index)) { if (tp == &PyRangeIter_Type) { specialize(instr, FOR_ITER_RANGE); return; @@ -2930,6 +2935,12 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT } else { if (tp == &PyList_Type) { +#ifdef Py_GIL_DISABLED + // Only specialize for lists owned by this thread or shared + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + goto failure; + } +#endif specialize(instr, FOR_ITER_LIST); return; } From 428735b5bc6d5095219e35114b26e65ec730d674 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 14:13:15 +0100 Subject: [PATCH 11/21] Add news --- .../2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst new file mode 100644 index 00000000000000..bfe2d633309191 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-30-14-13-01.gh-issue-132554.GqQaUp.rst @@ -0,0 +1,4 @@ +Change iteration to use "virtual iterators" for sequences. Instead of +creating an iterator, a tagged integer representing the next index is pushed +to the stack above the iterable. For non-sequence iterators, ``NULL`` is +pushed. From 4c83848fb32a585e4befdb8a5667831de9e6e58b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 17:12:00 +0100 Subject: [PATCH 12/21] handle tagged ints when doing type checks --- Include/internal/pycore_stackref.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index ad3699fef1d58c..211f65afde1457 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -714,18 +714,27 @@ PyStackRef_TYPE(_PyStackRef stackref) { static inline bool PyStackRef_GenCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); } static inline bool PyStackRef_BoolCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); } static inline bool PyStackRef_LongCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); } @@ -738,12 +747,18 @@ PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) static inline bool PyStackRef_CodeCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); } static inline bool PyStackRef_FunctionCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); } From f43ccc3b46b7ff29bb48b81e9a5eff69340b55d7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 17:22:32 +0100 Subject: [PATCH 13/21] Fix long check --- Include/internal/pycore_stackref.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 211f65afde1457..83e8b84f0622e8 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -733,7 +733,7 @@ static inline bool PyStackRef_LongCheck(_PyStackRef stackref) { if (PyStackRef_IsTaggedInt(stackref)) { - return false; + return true; } return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); } From 3fd46c3dcd22759805d36925b35e36ada1371c13 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 18:31:47 +0100 Subject: [PATCH 14/21] Attempt to make _PyForIter_NextWithIndex thread safe. --- Include/internal/pycore_stackref.h | 3 +++ Python/ceval.c | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 83e8b84f0622e8..7f3a5725a8dd23 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -741,6 +741,9 @@ PyStackRef_LongCheck(_PyStackRef stackref) static inline bool PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return false; + } return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); } diff --git a/Python/ceval.c b/Python/ceval.c index 0805b589cd7320..a8a8340f0f5f35 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3421,14 +3421,28 @@ _PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index) { assert(PyStackRef_IsTaggedInt(index)); - assert(Py_TYPE(seq) == &PyTuple_Type || Py_TYPE(seq) == &PyList_Type); - size_t size = Py_SIZE(seq); + assert(PyTuple_CheckExact(seq) || PyList_CheckExact(seq)); intptr_t i = PyStackRef_UntagInt(index); + if (PyTuple_CheckExact(seq)) { + size_t size = PyTuple_GET_SIZE(seq); + if ((size_t)i >= size) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, i)); + } + size_t size = PyList_GET_SIZE(seq); if ((size_t)i >= size) { return PyStackRef_NULL; } - PyObject *next_o = PySequence_Fast_GET_ITEM(seq, i); - return PyStackRef_FromPyObjectNew(next_o); +#ifdef Py_GIL_DISABLED + PyObject *item = _PyList_GetItemRef((PyListObject *)seq, i); + if (item == NULL) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectSteal(item); +#else + return PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, i)); +#endif } /* Check if a 'cls' provides the given special method. */ From 433282fbae085cee4722e8abd01c73cf07dbb4e9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 30 Apr 2025 20:20:39 +0100 Subject: [PATCH 15/21] Add review comment --- Python/specialize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/specialize.c b/Python/specialize.c index 1da692ebe065ef..44fce2bebd851c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2927,8 +2927,9 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR ); /* Don't specialize if PEP 523 is active */ - if (_PyInterpreterState_GET()->eval_frame) + if (_PyInterpreterState_GET()->eval_frame) { goto failure; + } specialize(instr, FOR_ITER_GEN); return; } From e915a05ce2261ee9edf9bf482e2719a6240f1b83 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 May 2025 10:55:30 +0100 Subject: [PATCH 16/21] Simplify list indexing code --- Python/ceval.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index a8a8340f0f5f35..30b195ae47cab3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3430,19 +3430,11 @@ _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index) } return PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, i)); } - size_t size = PyList_GET_SIZE(seq); - if ((size_t)i >= size) { - return PyStackRef_NULL; - } -#ifdef Py_GIL_DISABLED PyObject *item = _PyList_GetItemRef((PyListObject *)seq, i); if (item == NULL) { return PyStackRef_NULL; } return PyStackRef_FromPyObjectSteal(item); -#else - return PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, i)); -#endif } /* Check if a 'cls' provides the given special method. */ From 6d1b93e760b1312c2b938f89c87066be6195b303 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 May 2025 15:36:08 +0100 Subject: [PATCH 17/21] GET_ITER may leave the iterable on the stack --- Lib/test/test_list.py | 14 ++++++++++++++ Python/flowgraph.c | 1 + 2 files changed, 15 insertions(+) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 6894fba2ad1b00..450fc9024e4bac 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -365,5 +365,19 @@ def lappend(l, x, y): rc, _, _ = assert_python_ok("-c", code) self.assertEqual(rc, 0) + def test_list_overwrite_local(self): + """Test that overwriting the last reference to the + iterable doesn't prematurely free the iterable""" + + def foo(x): + r = 0 + for i in x: + r += i + x = None + return r + + self.assertEqual(foo(list(range(10))), 45) + + if __name__ == "__main__": unittest.main() diff --git a/Python/flowgraph.c b/Python/flowgraph.c index dabaf9bdc7cbb6..67ccf350b72ed6 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2871,6 +2871,7 @@ optimize_load_fast(cfg_builder *g) // Opcodes that consume no inputs case GET_ANEXT: + case GET_ITER: case GET_LEN: case IMPORT_FROM: case MATCH_KEYS: From 37ca28565accb04e6be039bde9b72c7cf25ae8f5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 May 2025 15:49:41 +0100 Subject: [PATCH 18/21] Fix test_dis --- Lib/test/test_dis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 2968b910110fb1..7967c9d9886962 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -851,7 +851,7 @@ def foo(x): %4d RETURN_GENERATOR POP_TOP L1: RESUME 0 - LOAD_FAST_BORROW 0 (.0) + LOAD_FAST 0 (.0) GET_ITER L2: FOR_ITER 14 (to L3) STORE_FAST 1 (z) From d9dc597a713c3fa72557520796b65b94ecea7189 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 27 May 2025 12:15:08 +0100 Subject: [PATCH 19/21] Clarify assert --- Include/internal/pycore_stackref.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index e7b9d1cdd01660..432c3675d02f6e 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -242,6 +242,7 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); #define Py_INT_TAG 3 #define Py_TAG_REFCNT 1 +#define Py_TAG_BITS 3 static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) @@ -268,7 +269,8 @@ PyStackRef_UntagInt(_PyStackRef i) static inline _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) { - assert(ref.bits != (uintptr_t)-1); // Deosn't overflow + assert((ref.bits & Py_TAG_BITS) == Py_INT_TAG); // Is tagged int + assert((ref.bits & (~Py_TAG_BITS)) != (INT_MAX & (~Py_TAG_BITS))); // Isn't about to overflow return (_PyStackRef){ .bits = ref.bits + 4 }; } @@ -278,7 +280,6 @@ PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) #define Py_TAG_DEFERRED Py_TAG_REFCNT #define Py_TAG_PTR ((uintptr_t)0) -#define Py_TAG_BITS ((uintptr_t)1) static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; @@ -453,7 +454,6 @@ PyStackRef_AsStrongReference(_PyStackRef stackref) /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT * as they can (must) have their reclamation deferred */ -#define Py_TAG_BITS 3 #if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT # error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT" #endif From d60ef2341735d50e710e67340f29f17a53b6051e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 27 May 2025 12:31:08 +0100 Subject: [PATCH 20/21] Checked for tagged ints --- Include/internal/pycore_stackref.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 432c3675d02f6e..c5fd61350cea15 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -247,7 +247,7 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) { - return (i.bits & Py_INT_TAG) == Py_INT_TAG; + return (i.bits & Py_TAG_BITS) == Py_INT_TAG; } static inline _PyStackRef @@ -274,6 +274,7 @@ PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) return (_PyStackRef){ .bits = ref.bits + 4 }; } +#define PyStackRef_IsDeferredOrTaggedInt(ref) (((ref).bits & Py_TAG_REFCNT) != 0) #ifdef Py_GIL_DISABLED @@ -407,7 +408,7 @@ static inline _PyStackRef PyStackRef_DUP(_PyStackRef stackref) { assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferred(stackref)) { + if (PyStackRef_IsDeferredOrTaggedInt(stackref)) { return stackref; } Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); From 9a9d20d1526e09e9b108edc0da1e0b11a368838d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 27 May 2025 14:08:37 +0100 Subject: [PATCH 21/21] Address review comments --- Include/internal/pycore_stackref.h | 56 +++++++----------------------- Lib/test/test_list.py | 1 + Python/stackrefs.c | 2 +- 3 files changed, 15 insertions(+), 44 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index c5fd61350cea15..f2ecc30b053568 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -392,7 +392,7 @@ PyStackRef_FromPyObjectBorrow(PyObject *obj) do { \ _PyStackRef _close_tmp = (REF); \ assert(!PyStackRef_IsNull(_close_tmp)); \ - if (!PyStackRef_IsDeferred(_close_tmp)) { \ + if (!PyStackRef_IsDeferredOrTaggedInt(_close_tmp)) { \ Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ } \ } while (0) @@ -704,23 +704,20 @@ PyStackRef_TYPE(_PyStackRef stackref) { // StackRef type checks -static inline bool -PyStackRef_GenCheck(_PyStackRef stackref) -{ - if (PyStackRef_IsTaggedInt(stackref)) { - return false; +#define STACKREF_CHECK_FUNC(T) \ + static inline bool \ + PyStackRef_ ## T ## Check(_PyStackRef stackref) { \ + if (PyStackRef_IsTaggedInt(stackref)) { \ + return false; \ + } \ + return Py ## T ## _Check(PyStackRef_AsPyObjectBorrow(stackref)); \ } - return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} -static inline bool -PyStackRef_BoolCheck(_PyStackRef stackref) -{ - if (PyStackRef_IsTaggedInt(stackref)) { - return false; - } - return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} +STACKREF_CHECK_FUNC(Gen) +STACKREF_CHECK_FUNC(Bool) +STACKREF_CHECK_FUNC(ExceptionInstance) +STACKREF_CHECK_FUNC(Code) +STACKREF_CHECK_FUNC(Function) static inline bool PyStackRef_LongCheck(_PyStackRef stackref) @@ -731,33 +728,6 @@ PyStackRef_LongCheck(_PyStackRef stackref) return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); } -static inline bool -PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) -{ - if (PyStackRef_IsTaggedInt(stackref)) { - return false; - } - return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - -static inline bool -PyStackRef_CodeCheck(_PyStackRef stackref) -{ - if (PyStackRef_IsTaggedInt(stackref)) { - return false; - } - return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - -static inline bool -PyStackRef_FunctionCheck(_PyStackRef stackref) -{ - if (PyStackRef_IsTaggedInt(stackref)) { - return false; - } - return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - static inline void _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) { diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 450fc9024e4bac..223f34fb696a6e 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -370,6 +370,7 @@ def test_list_overwrite_local(self): iterable doesn't prematurely free the iterable""" def foo(x): + self.assertEqual(sys.getrefcount(x), 1) r = 0 for i in x: r += i diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 9d0ef8d10f3f37..b2a1369031ad2b 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -219,7 +219,7 @@ PyStackRef_IsNullOrInt(_PyStackRef ref) _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) { - assert(ref.index != (uintptr_t)-1); // Overflow + assert(ref.index <= INT_MAX - 2); // No overflow return (_PyStackRef){ .index = ref.index + 2 }; } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy