From a7c92fc8b9c8c4c59f943732b4be49b22b714e3e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 17 Jun 2025 16:43:18 +0100 Subject: [PATCH 1/3] Specialize int operations for compact ints only. Deopts before allocation in case of overflow --- Include/cpython/longintrepr.h | 6 + Include/internal/pycore_long.h | 7 +- Include/internal/pycore_opcode_metadata.h | 8 +- Include/internal/pycore_optimizer.h | 10 + Include/internal/pycore_uop_ids.h | 280 +++++++++++----------- Include/internal/pycore_uop_metadata.h | 16 +- Lib/test/test_capi/test_opt.py | 6 +- Objects/longobject.c | 45 +++- Python/bytecodes.c | 43 ++-- Python/executor_cases.c.h | 77 +++--- Python/generated_cases.c.h | 69 ++---- Python/optimizer_analysis.c | 3 + Python/optimizer_bytecodes.c | 36 ++- Python/optimizer_cases.c.h | 41 +++- Python/optimizer_symbols.c | 114 ++++++++- Tools/cases_generator/analyzer.py | 1 + 16 files changed, 479 insertions(+), 283 deletions(-) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 4b6f97a5e475d6..9f12298d4fcc37 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -124,6 +124,12 @@ _PyLong_IsCompact(const PyLongObject* op) { return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS); } +static inline int +PyLong_CheckCompact(const PyObject *op) +{ + return PyLong_CheckExact(op) && _PyLong_IsCompact((PyLongObject *)op); +} + #define PyUnstable_Long_IsCompact _PyLong_IsCompact static inline Py_ssize_t diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index b8efba74bdc421..852d22bb8c5749 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -112,9 +112,9 @@ PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); -PyAPI_FUNC(PyObject*) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right); -PyAPI_FUNC(PyObject*) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right); -PyAPI_FUNC(PyObject*) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(_PyStackRef) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(_PyStackRef) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(_PyStackRef) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; @@ -213,7 +213,6 @@ _PyLong_BothAreCompact(const PyLongObject* a, const PyLongObject* b) { assert(PyLong_Check(b)); return (a->long_value.lv_tag | b->long_value.lv_tag) < (2 << NON_SIZE_BITS); } - static inline bool _PyLong_IsZero(const PyLongObject *op) { diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 59c85ddeb3d663..dd1bf2d1d2b51a 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1072,12 +1072,12 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1085,7 +1085,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1129,7 +1129,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d3674726997f6a..71bbb8ec0affcc 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -178,6 +178,7 @@ typedef enum _JitSymType { JIT_SYM_KNOWN_VALUE_TAG = 7, JIT_SYM_TUPLE_TAG = 8, JIT_SYM_TRUTHINESS_TAG = 9, + JIT_SYM_COMPACT_INT = 10, } JitSymType; typedef struct _jit_opt_known_class { @@ -210,6 +211,10 @@ typedef struct { uint16_t value; } JitOptTruthiness; +typedef struct { + uint8_t tag; +} JitOptCompactInt; + typedef union _jit_opt_symbol { uint8_t tag; JitOptKnownClass cls; @@ -217,6 +222,7 @@ typedef union _jit_opt_symbol { JitOptKnownVersion version; JitOptTuple tuple; JitOptTruthiness truthiness; + JitOptCompactInt compact; } JitOptSymbol; @@ -282,6 +288,10 @@ extern JitOptSymbol *_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptS extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item); extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); +extern bool _Py_uop_sym_is_compact_int(JitOptSymbol *sym); +extern JitOptSymbol *_Py_uop_sym_new_compact_int(JitOptContext *ctx); +extern void _Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptSymbol *value); + extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8211c5d056535e..2a9ca77cdc99db 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -143,32 +143,34 @@ extern "C" { #define _GUARD_NOS_LIST 398 #define _GUARD_NOS_NOT_NULL 399 #define _GUARD_NOS_NULL 400 -#define _GUARD_NOS_TUPLE 401 -#define _GUARD_NOS_UNICODE 402 -#define _GUARD_NOT_EXHAUSTED_LIST 403 -#define _GUARD_NOT_EXHAUSTED_RANGE 404 -#define _GUARD_NOT_EXHAUSTED_TUPLE 405 -#define _GUARD_THIRD_NULL 406 -#define _GUARD_TOS_ANY_SET 407 -#define _GUARD_TOS_DICT 408 -#define _GUARD_TOS_FLOAT 409 -#define _GUARD_TOS_INT 410 -#define _GUARD_TOS_LIST 411 -#define _GUARD_TOS_SLICE 412 -#define _GUARD_TOS_TUPLE 413 -#define _GUARD_TOS_UNICODE 414 -#define _GUARD_TYPE_VERSION 415 -#define _GUARD_TYPE_VERSION_AND_LOCK 416 +#define _GUARD_NOS_OVERFLOWED 401 +#define _GUARD_NOS_TUPLE 402 +#define _GUARD_NOS_UNICODE 403 +#define _GUARD_NOT_EXHAUSTED_LIST 404 +#define _GUARD_NOT_EXHAUSTED_RANGE 405 +#define _GUARD_NOT_EXHAUSTED_TUPLE 406 +#define _GUARD_THIRD_NULL 407 +#define _GUARD_TOS_ANY_SET 408 +#define _GUARD_TOS_DICT 409 +#define _GUARD_TOS_FLOAT 410 +#define _GUARD_TOS_INT 411 +#define _GUARD_TOS_LIST 412 +#define _GUARD_TOS_OVERFLOWED 413 +#define _GUARD_TOS_SLICE 414 +#define _GUARD_TOS_TUPLE 415 +#define _GUARD_TOS_UNICODE 416 +#define _GUARD_TYPE_VERSION 417 +#define _GUARD_TYPE_VERSION_AND_LOCK 418 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 417 -#define _INIT_CALL_PY_EXACT_ARGS 418 -#define _INIT_CALL_PY_EXACT_ARGS_0 419 -#define _INIT_CALL_PY_EXACT_ARGS_1 420 -#define _INIT_CALL_PY_EXACT_ARGS_2 421 -#define _INIT_CALL_PY_EXACT_ARGS_3 422 -#define _INIT_CALL_PY_EXACT_ARGS_4 423 -#define _INSERT_NULL 424 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 419 +#define _INIT_CALL_PY_EXACT_ARGS 420 +#define _INIT_CALL_PY_EXACT_ARGS_0 421 +#define _INIT_CALL_PY_EXACT_ARGS_1 422 +#define _INIT_CALL_PY_EXACT_ARGS_2 423 +#define _INIT_CALL_PY_EXACT_ARGS_3 424 +#define _INIT_CALL_PY_EXACT_ARGS_4 425 +#define _INSERT_NULL 426 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -178,173 +180,173 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 425 +#define _IS_NONE 427 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 426 -#define _ITER_CHECK_RANGE 427 -#define _ITER_CHECK_TUPLE 428 -#define _ITER_JUMP_LIST 429 -#define _ITER_JUMP_RANGE 430 -#define _ITER_JUMP_TUPLE 431 -#define _ITER_NEXT_LIST 432 -#define _ITER_NEXT_LIST_TIER_TWO 433 -#define _ITER_NEXT_RANGE 434 -#define _ITER_NEXT_TUPLE 435 -#define _JUMP_TO_TOP 436 +#define _ITER_CHECK_LIST 428 +#define _ITER_CHECK_RANGE 429 +#define _ITER_CHECK_TUPLE 430 +#define _ITER_JUMP_LIST 431 +#define _ITER_JUMP_RANGE 432 +#define _ITER_JUMP_TUPLE 433 +#define _ITER_NEXT_LIST 434 +#define _ITER_NEXT_LIST_TIER_TWO 435 +#define _ITER_NEXT_RANGE 436 +#define _ITER_NEXT_TUPLE 437 +#define _JUMP_TO_TOP 438 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 437 -#define _LOAD_ATTR_CLASS 438 +#define _LOAD_ATTR 439 +#define _LOAD_ATTR_CLASS 440 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 439 -#define _LOAD_ATTR_METHOD_LAZY_DICT 440 -#define _LOAD_ATTR_METHOD_NO_DICT 441 -#define _LOAD_ATTR_METHOD_WITH_VALUES 442 -#define _LOAD_ATTR_MODULE 443 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 444 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 445 -#define _LOAD_ATTR_PROPERTY_FRAME 446 -#define _LOAD_ATTR_SLOT 447 -#define _LOAD_ATTR_WITH_HINT 448 +#define _LOAD_ATTR_INSTANCE_VALUE 441 +#define _LOAD_ATTR_METHOD_LAZY_DICT 442 +#define _LOAD_ATTR_METHOD_NO_DICT 443 +#define _LOAD_ATTR_METHOD_WITH_VALUES 444 +#define _LOAD_ATTR_MODULE 445 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 446 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 447 +#define _LOAD_ATTR_PROPERTY_FRAME 448 +#define _LOAD_ATTR_SLOT 449 +#define _LOAD_ATTR_WITH_HINT 450 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 449 +#define _LOAD_BYTECODE 451 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 450 -#define _LOAD_CONST_INLINE_BORROW 451 -#define _LOAD_CONST_UNDER_INLINE 452 -#define _LOAD_CONST_UNDER_INLINE_BORROW 453 +#define _LOAD_CONST_INLINE 452 +#define _LOAD_CONST_INLINE_BORROW 453 +#define _LOAD_CONST_UNDER_INLINE 454 +#define _LOAD_CONST_UNDER_INLINE_BORROW 455 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 454 -#define _LOAD_FAST_0 455 -#define _LOAD_FAST_1 456 -#define _LOAD_FAST_2 457 -#define _LOAD_FAST_3 458 -#define _LOAD_FAST_4 459 -#define _LOAD_FAST_5 460 -#define _LOAD_FAST_6 461 -#define _LOAD_FAST_7 462 +#define _LOAD_FAST 456 +#define _LOAD_FAST_0 457 +#define _LOAD_FAST_1 458 +#define _LOAD_FAST_2 459 +#define _LOAD_FAST_3 460 +#define _LOAD_FAST_4 461 +#define _LOAD_FAST_5 462 +#define _LOAD_FAST_6 463 +#define _LOAD_FAST_7 464 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 463 -#define _LOAD_FAST_BORROW_0 464 -#define _LOAD_FAST_BORROW_1 465 -#define _LOAD_FAST_BORROW_2 466 -#define _LOAD_FAST_BORROW_3 467 -#define _LOAD_FAST_BORROW_4 468 -#define _LOAD_FAST_BORROW_5 469 -#define _LOAD_FAST_BORROW_6 470 -#define _LOAD_FAST_BORROW_7 471 +#define _LOAD_FAST_BORROW 465 +#define _LOAD_FAST_BORROW_0 466 +#define _LOAD_FAST_BORROW_1 467 +#define _LOAD_FAST_BORROW_2 468 +#define _LOAD_FAST_BORROW_3 469 +#define _LOAD_FAST_BORROW_4 470 +#define _LOAD_FAST_BORROW_5 471 +#define _LOAD_FAST_BORROW_6 472 +#define _LOAD_FAST_BORROW_7 473 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 472 -#define _LOAD_GLOBAL_BUILTINS 473 -#define _LOAD_GLOBAL_MODULE 474 +#define _LOAD_GLOBAL 474 +#define _LOAD_GLOBAL_BUILTINS 475 +#define _LOAD_GLOBAL_MODULE 476 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 475 -#define _LOAD_SMALL_INT_0 476 -#define _LOAD_SMALL_INT_1 477 -#define _LOAD_SMALL_INT_2 478 -#define _LOAD_SMALL_INT_3 479 -#define _LOAD_SPECIAL 480 +#define _LOAD_SMALL_INT 477 +#define _LOAD_SMALL_INT_0 478 +#define _LOAD_SMALL_INT_1 479 +#define _LOAD_SMALL_INT_2 480 +#define _LOAD_SMALL_INT_3 481 +#define _LOAD_SPECIAL 482 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 481 +#define _MAKE_CALLARGS_A_TUPLE 483 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 482 +#define _MAKE_WARM 484 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 483 -#define _MAYBE_EXPAND_METHOD_KW 484 -#define _MONITOR_CALL 485 -#define _MONITOR_CALL_KW 486 -#define _MONITOR_JUMP_BACKWARD 487 -#define _MONITOR_RESUME 488 +#define _MAYBE_EXPAND_METHOD 485 +#define _MAYBE_EXPAND_METHOD_KW 486 +#define _MONITOR_CALL 487 +#define _MONITOR_CALL_KW 488 +#define _MONITOR_JUMP_BACKWARD 489 +#define _MONITOR_RESUME 490 #define _NOP NOP -#define _POP_CALL 489 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 490 -#define _POP_CALL_ONE 491 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 492 -#define _POP_CALL_TWO 493 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 494 +#define _POP_CALL 491 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 492 +#define _POP_CALL_ONE 493 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 494 +#define _POP_CALL_TWO 495 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 496 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 495 -#define _POP_JUMP_IF_TRUE 496 +#define _POP_JUMP_IF_FALSE 497 +#define _POP_JUMP_IF_TRUE 498 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 497 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 498 -#define _POP_TWO 499 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 500 +#define _POP_TOP_LOAD_CONST_INLINE 499 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 500 +#define _POP_TWO 501 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 502 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 501 +#define _PUSH_FRAME 503 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 502 -#define _PY_FRAME_GENERAL 503 -#define _PY_FRAME_KW 504 -#define _QUICKEN_RESUME 505 -#define _REPLACE_WITH_TRUE 506 +#define _PUSH_NULL_CONDITIONAL 504 +#define _PY_FRAME_GENERAL 505 +#define _PY_FRAME_KW 506 +#define _QUICKEN_RESUME 507 +#define _REPLACE_WITH_TRUE 508 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 507 -#define _SEND 508 -#define _SEND_GEN_FRAME 509 +#define _SAVE_RETURN_OFFSET 509 +#define _SEND 510 +#define _SEND_GEN_FRAME 511 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 510 -#define _STORE_ATTR 511 -#define _STORE_ATTR_INSTANCE_VALUE 512 -#define _STORE_ATTR_SLOT 513 -#define _STORE_ATTR_WITH_HINT 514 +#define _START_EXECUTOR 512 +#define _STORE_ATTR 513 +#define _STORE_ATTR_INSTANCE_VALUE 514 +#define _STORE_ATTR_SLOT 515 +#define _STORE_ATTR_WITH_HINT 516 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 515 -#define _STORE_FAST_0 516 -#define _STORE_FAST_1 517 -#define _STORE_FAST_2 518 -#define _STORE_FAST_3 519 -#define _STORE_FAST_4 520 -#define _STORE_FAST_5 521 -#define _STORE_FAST_6 522 -#define _STORE_FAST_7 523 +#define _STORE_FAST 517 +#define _STORE_FAST_0 518 +#define _STORE_FAST_1 519 +#define _STORE_FAST_2 520 +#define _STORE_FAST_3 521 +#define _STORE_FAST_4 522 +#define _STORE_FAST_5 523 +#define _STORE_FAST_6 524 +#define _STORE_FAST_7 525 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 524 -#define _STORE_SUBSCR 525 -#define _STORE_SUBSCR_DICT 526 -#define _STORE_SUBSCR_LIST_INT 527 -#define _SWAP 528 -#define _SWAP_2 529 -#define _SWAP_3 530 -#define _TIER2_RESUME_CHECK 531 -#define _TO_BOOL 532 +#define _STORE_SLICE 526 +#define _STORE_SUBSCR 527 +#define _STORE_SUBSCR_DICT 528 +#define _STORE_SUBSCR_LIST_INT 529 +#define _SWAP 530 +#define _SWAP_2 531 +#define _SWAP_3 532 +#define _TIER2_RESUME_CHECK 533 +#define _TO_BOOL 534 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 533 +#define _TO_BOOL_LIST 535 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 534 +#define _TO_BOOL_STR 536 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 535 -#define _UNPACK_SEQUENCE_LIST 536 -#define _UNPACK_SEQUENCE_TUPLE 537 -#define _UNPACK_SEQUENCE_TWO_TUPLE 538 +#define _UNPACK_SEQUENCE 537 +#define _UNPACK_SEQUENCE_LIST 538 +#define _UNPACK_SEQUENCE_TUPLE 539 +#define _UNPACK_SEQUENCE_TWO_TUPLE 540 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 538 +#define MAX_UOP_ID 540 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index fad87d4b586e64..1e8c48aed53dfd 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -86,9 +86,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, + [_GUARD_TOS_OVERFLOWED] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, @@ -185,7 +187,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, - [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -469,6 +471,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", [_GUARD_NOS_NOT_NULL] = "_GUARD_NOS_NOT_NULL", [_GUARD_NOS_NULL] = "_GUARD_NOS_NULL", + [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED", [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", @@ -480,6 +483,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", + [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", [_GUARD_TOS_SLICE] = "_GUARD_TOS_SLICE", [_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE", [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", @@ -783,6 +787,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TOS_INT: return 0; + case _GUARD_NOS_OVERFLOWED: + return 0; + case _GUARD_TOS_OVERFLOWED: + return 0; case _BINARY_OP_MULTIPLY_INT: return 2; case _BINARY_OP_ADD_INT: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 3fc2cb33795a5f..949e1f9039eb3d 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -407,12 +407,12 @@ def testfunc(n, m): x = 0 for i in range(m): for j in MyIter(n): - x += 1000*i + j + x += j return x - x = testfunc(TIER2_THRESHOLD, TIER2_THRESHOLD) + x = testfunc(TIER2_THRESHOLD, 2) - self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * TIER2_THRESHOLD * 1001) + self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * 2) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) diff --git a/Objects/longobject.c b/Objects/longobject.c index dfa02851cd8877..59b10355ad9df8 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -10,6 +10,7 @@ #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS +#include "pycore_stackref.h" #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include "pycore_unicodeobject.h" // _PyUnicode_Equal() @@ -316,6 +317,33 @@ _PyLong_FromSTwoDigits(stwodigits x) return (PyLongObject*)_PyLong_FromLarge(x); } +/* Create a new medium int object from a medium int. + * Do not raise. Return NULL if not medium or can't allocate. */ +static inline _PyStackRef +medium_from_stwodigits(stwodigits x) +{ + if (IS_SMALL_INT(x)) { + return PyStackRef_FromPyObjectBorrow(get_small_int((sdigit)x)); + } + assert(x != 0); + if(!is_medium_int(x)) { + return PyStackRef_NULL; + } + PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); + if (v == NULL) { + v = PyObject_Malloc(sizeof(PyLongObject)); + if (v == NULL) { + return PyStackRef_NULL; + } + _PyObject_Init((PyObject*)v, &PyLong_Type); + } + digit abs_x = x < 0 ? -x : x; + _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); + v->long_value.ob_digit[0] = abs_x; + return PyStackRef_FromPyObjectStealMortal((PyObject *)v); +} + + /* If a freshly-allocated int is already shared, it must be a small integer, so negating it must go to PyLong_FromLong */ Py_LOCAL_INLINE(void) @@ -3771,12 +3799,12 @@ long_add(PyLongObject *a, PyLongObject *b) return z; } -PyObject * +_PyStackRef _PyCompactLong_Add(PyLongObject *a, PyLongObject *b) { assert(_PyLong_BothAreCompact(a, b)); - stwodigits z = medium_value(a) + medium_value(b); - return (PyObject *)_PyLong_FromSTwoDigits(z); + stwodigits v = medium_value(a) + medium_value(b); + return medium_from_stwodigits(v); } static PyObject * @@ -3816,11 +3844,12 @@ long_sub(PyLongObject *a, PyLongObject *b) return z; } -PyObject * +_PyStackRef _PyCompactLong_Subtract(PyLongObject *a, PyLongObject *b) { assert(_PyLong_BothAreCompact(a, b)); - return (PyObject *)_PyLong_FromSTwoDigits(medium_value(a) - medium_value(b)); + stwodigits v = medium_value(a) - medium_value(b); + return medium_from_stwodigits(v); } static PyObject * @@ -4264,12 +4293,14 @@ long_mul(PyLongObject *a, PyLongObject *b) return z; } -PyObject * +/* This function returns NULL if the result is not compact, + * or if it fails to allocate, but never raises */ +_PyStackRef _PyCompactLong_Multiply(PyLongObject *a, PyLongObject *b) { assert(_PyLong_BothAreCompact(a, b)); stwodigits v = medium_value(a) * medium_value(b); - return (PyObject *)_PyLong_FromSTwoDigits(v); + return medium_from_stwodigits(v); } static PyObject * diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 27a04766cc8dd8..d6fc5e3e86aa28 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -569,12 +569,24 @@ dummy_func( op(_GUARD_NOS_INT, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - EXIT_IF(!PyLong_CheckExact(left_o)); + EXIT_IF(!PyLong_CheckCompact(left_o)); } op(_GUARD_TOS_INT, (value -- value)) { PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - EXIT_IF(!PyLong_CheckExact(value_o)); + EXIT_IF(!PyLong_CheckCompact(value_o)); + } + + op(_GUARD_NOS_OVERFLOWED, (left, unused -- left, unused)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + EXIT_IF(!_PyLong_IsCompact((PyLongObject *)left_o)); + } + + op(_GUARD_TOS_OVERFLOWED, (value -- value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + EXIT_IF(!_PyLong_IsCompact((PyLongObject *)value_o)); } pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { @@ -582,15 +594,14 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + EXIT_IF(PyStackRef_IsNull(res)); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { @@ -598,15 +609,14 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + EXIT_IF(PyStackRef_IsNull(res)); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { @@ -614,21 +624,22 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - DEOPT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + EXIT_IF(PyStackRef_IsNull(res)); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_MULTIPLY_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT; + macro(BINARY_OP_ADD_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT; + macro(BINARY_OP_SUBTRACT_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT; @@ -2691,8 +2702,8 @@ dummy_func( PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o)); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o)); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 74c78e4d1f5a69..82eb0102998b9d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -852,7 +852,7 @@ _PyStackRef left; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!PyLong_CheckCompact(left_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -863,7 +863,31 @@ _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_NOS_OVERFLOWED: { + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_OVERFLOWED: { + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -880,20 +904,15 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -910,20 +929,15 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -940,20 +954,15 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3726,14 +3735,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (!_PyLong_IsCompact((PyLongObject *)right_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4fc1d5266d0a87..82142aee9a3aea 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -158,7 +158,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -168,7 +168,7 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!PyLong_CheckCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -182,19 +182,16 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -486,7 +483,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -496,7 +493,7 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!PyLong_CheckCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -510,19 +507,16 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -700,7 +694,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -862,7 +856,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -940,7 +934,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1070,7 +1064,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1080,7 +1074,7 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!PyLong_CheckCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1094,19 +1088,16 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -4902,7 +4893,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -4912,7 +4903,7 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!PyLong_CheckCompact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -4924,16 +4915,8 @@ right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); - } - if (!_PyLong_IsCompact((PyLongObject *)right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); - } + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -11490,7 +11473,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!PyLong_CheckCompact(value_o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6a7df233819b9c..4498dba8b1ff5e 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -333,6 +333,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM) #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new @@ -341,6 +342,8 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_tuple_getitem _Py_uop_sym_tuple_getitem #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_is_compact_int _Py_uop_sym_is_compact_int +#define sym_new_compact_int _Py_uop_sym_new_compact_int #define sym_new_truthiness _Py_uop_sym_new_truthiness static int diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 5a9a3a943a7b02..93ea3cef5bf03d 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -27,6 +27,7 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM) #define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop @@ -34,6 +35,8 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_tuple_getitem _Py_uop_sym_tuple_getitem #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_new_compact_int _Py_uop_sym_new_compact_int +#define sym_is_compact_int _Py_uop_sym_is_compact_int #define sym_new_truthiness _Py_uop_sym_new_truthiness extern int @@ -105,17 +108,27 @@ dummy_func(void) { } op(_GUARD_TOS_INT, (value -- value)) { - if (sym_matches_type(value, &PyLong_Type)) { + if (sym_is_compact_int(value)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyLong_Type); + else { + if (sym_get_type(value) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(value); + } } op(_GUARD_NOS_INT, (left, unused -- left, unused)) { - if (sym_matches_type(left, &PyLong_Type)) { + if (sym_is_compact_int(left)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); + else { + if (sym_get_type(left) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(left); + } } op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { @@ -222,15 +235,15 @@ dummy_func(void) { } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - res = sym_new_type(ctx, &PyLong_Type); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - res = sym_new_type(ctx, &PyLong_Type); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - res = sym_new_type(ctx, &PyLong_Type); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { @@ -422,6 +435,15 @@ dummy_func(void) { res = sym_new_truthiness(ctx, value, false); } + op(_UNARY_NEGATIVE, (value -- res)) { + if (sym_is_compact_int(value)) { + res = sym_new_compact_int(ctx); + } + else { + res = sym_new_not_null(ctx); + } + } + op(_UNARY_INVERT, (value -- res)) { if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8c4f0399c75a73..c10d130ce80bad 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -142,8 +142,15 @@ } case _UNARY_NEGATIVE: { + JitOptSymbol *value; JitOptSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + if (sym_is_compact_int(value)) { + res = sym_new_compact_int(ctx); + } + else { + res = sym_new_not_null(ctx); + } stack_pointer[-1] = res; break; } @@ -301,26 +308,44 @@ case _GUARD_NOS_INT: { JitOptSymbol *left; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type)) { + if (sym_is_compact_int(left)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); + else { + if (sym_get_type(left) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(left); + } break; } case _GUARD_TOS_INT: { JitOptSymbol *value; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyLong_Type)) { + if (sym_is_compact_int(value)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyLong_Type); + else { + if (sym_get_type(value) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(value); + } + break; + } + + case _GUARD_NOS_OVERFLOWED: { + break; + } + + case _GUARD_TOS_OVERFLOWED: { break; } case _BINARY_OP_MULTIPLY_INT: { JitOptSymbol *res; - res = sym_new_type(ctx, &PyLong_Type); + res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -329,7 +354,7 @@ case _BINARY_OP_ADD_INT: { JitOptSymbol *res; - res = sym_new_type(ctx, &PyLong_Type); + res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -338,7 +363,7 @@ case _BINARY_OP_SUBTRACT_INT: { JitOptSymbol *res; - res = sym_new_type(ctx, &PyLong_Type); + res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 25de5d83166f64..965b47c503eddc 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -30,17 +30,19 @@ we often skip in-between states for convenience: | | NULL | | | <- Anything below this level is an object. -| NON_NULL -| | | <- Anything below this level has a known type version. -| TYPE_VERSION | -| | | <- Anything below this level has a known type. -| KNOWN_CLASS | -| | | | <- Anything below this level has a known truthiness. -| | | TRUTHINESS -| | | | -| TUPLE | | -| | | | <- Anything below this level is a known constant. -| KNOWN_VALUE +| NON_NULL-+ +| | | <- Anything below this level has a known type version. +| TYPE_VERSION | +| | | <- Anything below this level has a known type. +| KNOWN_CLASS | +| | | | | | +| | | INT* | | +| | | | | | <- Anything below this level has a known truthiness. +| | | | | TRUTHINESS +| | | | | | +| TUPLE | | | | +| | | | | | <- Anything below this level is a known constant. +| KNOWN_VALUE--+ | | <- Anything below this level is unreachable. BOTTOM @@ -52,6 +54,8 @@ result of a truth test, which would allow us to narrow the symbol to KNOWN_VALUE the same symbol, that would be a contradiction, and the symbol would be set to BOTTOM (indicating that the code is unreachable). +INT* is a limited range int, currently a "compact" int. + */ #ifdef Py_DEBUG @@ -218,6 +222,11 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) sym_set_bottom(ctx, sym); } return; + case JIT_SYM_COMPACT_INT: + if (typ != &PyLong_Type) { + sym_set_bottom(ctx, sym); + } + return; } } @@ -274,6 +283,12 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int return false; } return true; + case JIT_SYM_COMPACT_INT: + if (version != PyLong_Type.tp_version_tag) { + sym_set_bottom(ctx, sym); + return false; + } + return true; } Py_UNREACHABLE(); } @@ -356,6 +371,14 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val // TODO: More types (GH-130415)! make_const(sym, const_val); return; + case JIT_SYM_COMPACT_INT: + if (PyLong_CheckCompact(const_val)) { + make_const(sym, const_val); + } + else { + sym_set_bottom(ctx, sym); + } + return; } } @@ -458,6 +481,9 @@ _Py_uop_sym_get_type(JitOptSymbol *sym) return &PyTuple_Type; case JIT_SYM_TRUTHINESS_TAG: return &PyBool_Type; + case JIT_SYM_COMPACT_INT: + return &PyLong_Type; + } Py_UNREACHABLE(); } @@ -482,6 +508,8 @@ _Py_uop_sym_get_type_version(JitOptSymbol *sym) return PyTuple_Type.tp_version_tag; case JIT_SYM_TRUTHINESS_TAG: return PyBool_Type.tp_version_tag; + case JIT_SYM_COMPACT_INT: + return PyLong_Type.tp_version_tag; } Py_UNREACHABLE(); } @@ -514,6 +542,7 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) case JIT_SYM_BOTTOM_TAG: case JIT_SYM_NON_NULL_TAG: case JIT_SYM_UNKNOWN_TAG: + case JIT_SYM_COMPACT_INT: return -1; case JIT_SYM_KNOWN_CLASS_TAG: /* TODO : @@ -621,6 +650,58 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) return false; } +bool +_Py_uop_sym_is_compact_int(JitOptSymbol *sym) +{ + if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { + return (bool)PyLong_CheckCompact(sym->value.value); + } + return sym->tag == JIT_SYM_COMPACT_INT; +} + +void +_Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptSymbol *sym) +{ + JitSymType tag = sym->tag; + switch(tag) { + case JIT_SYM_NULL_TAG: + sym_set_bottom(ctx, sym); + return; + case JIT_SYM_KNOWN_CLASS_TAG: + if (sym->cls.type == &PyLong_Type) { + sym->tag = JIT_SYM_COMPACT_INT; + } else { + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_TYPE_VERSION_TAG: + if (sym->version.version == PyLong_Type.tp_version_tag) { + sym->tag = JIT_SYM_COMPACT_INT; + } + else { + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_KNOWN_VALUE_TAG: + if (!PyLong_CheckCompact(sym->value.value)) { + Py_CLEAR(sym->value.value); + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_TUPLE_TAG: + case JIT_SYM_TRUTHINESS_TAG: + sym_set_bottom(ctx, sym); + return; + case JIT_SYM_BOTTOM_TAG: + case JIT_SYM_COMPACT_INT: + return; + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + sym->tag = JIT_SYM_COMPACT_INT; + return; + } +} + JitOptSymbol * _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) { @@ -645,6 +726,17 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) return res; } +JitOptSymbol * +_Py_uop_sym_new_compact_int(JitOptContext *ctx) +{ + JitOptSymbol *sym = sym_new(ctx); + if (sym == NULL) { + return out_of_space(ctx); + } + sym->tag = JIT_SYM_COMPACT_INT; + return sym; +} + // 0 on success, -1 on error. _Py_UOpsAbstractFrame * _Py_uop_frame_new( diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c6a9fbcad8891f..40d74357f801b8 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -687,6 +687,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_IsValid", "PyStackRef_Wrap", "PyStackRef_Unwrap", + "PyLong_CheckCompact", ) From e27d994929a1ab84205bb419afd6ccfee252265b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 18 Jun 2025 12:19:23 +0100 Subject: [PATCH 2/3] Add news --- .../2025-06-18-12-19-13.gh-issue-135379.TCvGpj.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-12-19-13.gh-issue-135379.TCvGpj.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-12-19-13.gh-issue-135379.TCvGpj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-12-19-13.gh-issue-135379.TCvGpj.rst new file mode 100644 index 00000000000000..089d00c77da25d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-12-19-13.gh-issue-135379.TCvGpj.rst @@ -0,0 +1,3 @@ +Changes specialization of ``BINARY_OP`` for ints to only specialize for +"compact" ints. This streamlines the fast path at the cost of fewer +specializations when very large integers are used. From 996899b86fc4dc7067852d862e4da628ad1ecdd6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 18 Jun 2025 14:30:31 +0100 Subject: [PATCH 3/3] Add tests --- Python/optimizer_symbols.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index e806b0893d7e92..8ae670d10280ec 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -890,6 +890,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_uop_abstractcontext_init(ctx); PyObject *val_42 = NULL; PyObject *val_43 = NULL; + PyObject *val_big = NULL; PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. @@ -1020,9 +1021,38 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == Py_False, "truthiness is not False"); TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); + + + val_big = PyNumber_Lshift(_PyLong_GetOne(), PyLong_FromLong(66)); + if (val_big == NULL) { + goto fail; + } + + JitOptRef ref_42 = _Py_uop_sym_new_const(ctx, val_42); + JitOptRef ref_big = _Py_uop_sym_new_const(ctx, val_big); + JitOptRef ref_int = _Py_uop_sym_new_compact_int(ctx); + TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_42), "42 is not a compact int"); + TEST_PREDICATE(!_Py_uop_sym_is_compact_int(ref_big), "(1 << 66) is a compact int"); + TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "compact int is not a compact int"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "compact int is not an int"); + + _Py_uop_sym_set_type(ctx, ref_int, &PyLong_Type); // Should have no effect + TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "compact int is not a compact int after cast"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "compact int is not an int after cast"); + + _Py_uop_sym_set_type(ctx, ref_int, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref_int), "compact int cast to float isn't bottom"); + + ref_int = _Py_uop_sym_new_compact_int(ctx); + _Py_uop_sym_set_const(ctx, ref_int, val_43); + TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "43 is not a compact int"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "43 is not an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref_int) == val_43, "43 isn't 43"); + _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); + Py_DECREF(val_big); Py_DECREF(tuple); Py_RETURN_NONE; @@ -1030,6 +1060,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_uop_abstractcontext_fini(ctx); Py_XDECREF(val_42); Py_XDECREF(val_43); + Py_XDECREF(val_big); Py_DECREF(tuple); return NULL; } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy