From 94664176280c6cf50c5014fa31f1c72021076930 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 23 May 2025 21:28:51 +0800 Subject: [PATCH 01/24] Skip refcounting where possible for common float ops --- Include/internal/pycore_optimizer.h | 41 ++- Include/internal/pycore_uop_ids.h | 461 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 12 + Lib/test/test_capi/test_opt.py | 20 +- Python/bytecodes.c | 46 +++ Python/executor_cases.c.h | 81 +++++ Python/optimizer_analysis.c | 10 + Python/optimizer_bytecodes.c | 16 + Python/optimizer_cases.c.h | 40 +++ Python/optimizer_symbols.c | 35 +- 10 files changed, 523 insertions(+), 239 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d3674726997f6a..e8b61096f477a1 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -180,38 +180,63 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; +#define DONT_SKIP_REFCOUNT 0 +#define SKIP_REFCOUNT 1 + typedef struct _jit_opt_known_class { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; uint32_t version; PyTypeObject *type; } JitOptKnownClass; typedef struct _jit_opt_known_version { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; uint32_t version; } JitOptKnownVersion; typedef struct _jit_opt_known_value { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; PyObject *value; } JitOptKnownValue; #define MAX_SYMBOLIC_TUPLE_SIZE 7 typedef struct _jit_opt_tuple { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; uint8_t length; uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; } JitOptTuple; typedef struct { - uint8_t tag; + struct { + uint8_t tag; + uint8_t skip_refcount; + }; bool invert; uint16_t value; } JitOptTruthiness; typedef union _jit_opt_symbol { - uint8_t tag; + struct { + uint8_t tag; + // Whether this object skips refcount on the stack + // (using the _PyStackRef API), or not. + // 0 - normal refcounting + // 1 - skip refcounting + int8_t skip_refcount; + }; JitOptKnownClass cls; JitOptKnownValue value; JitOptKnownVersion version; @@ -283,6 +308,10 @@ extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); +extern void _Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern bool _Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); + extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d08799487fda42..e48da341b63f1a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,22 +13,25 @@ extern "C" { #define _SET_IP 301 #define _BINARY_OP 302 #define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_EXTEND 306 -#define _BINARY_OP_INPLACE_ADD_UNICODE 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_MULTIPLY_INT 309 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 310 -#define _BINARY_OP_SUBSCR_DICT 311 -#define _BINARY_OP_SUBSCR_INIT_CALL 312 -#define _BINARY_OP_SUBSCR_LIST_INT 313 -#define _BINARY_OP_SUBSCR_LIST_SLICE 314 -#define _BINARY_OP_SUBSCR_STR_INT 315 -#define _BINARY_OP_SUBSCR_TUPLE_INT 316 -#define _BINARY_OP_SUBTRACT_FLOAT 317 -#define _BINARY_OP_SUBTRACT_INT 318 -#define _BINARY_SLICE 319 +#define _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF 304 +#define _BINARY_OP_ADD_INT 305 +#define _BINARY_OP_ADD_UNICODE 306 +#define _BINARY_OP_EXTEND 307 +#define _BINARY_OP_INPLACE_ADD_UNICODE 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF 310 +#define _BINARY_OP_MULTIPLY_INT 311 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 312 +#define _BINARY_OP_SUBSCR_DICT 313 +#define _BINARY_OP_SUBSCR_INIT_CALL 314 +#define _BINARY_OP_SUBSCR_LIST_INT 315 +#define _BINARY_OP_SUBSCR_LIST_SLICE 316 +#define _BINARY_OP_SUBSCR_STR_INT 317 +#define _BINARY_OP_SUBSCR_TUPLE_INT 318 +#define _BINARY_OP_SUBTRACT_FLOAT 319 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF 320 +#define _BINARY_OP_SUBTRACT_INT 321 +#define _BINARY_SLICE 322 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -37,135 +40,135 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 320 -#define _CALL_BUILTIN_FAST 321 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 322 -#define _CALL_BUILTIN_O 323 +#define _CALL_BUILTIN_CLASS 323 +#define _CALL_BUILTIN_FAST 324 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 +#define _CALL_BUILTIN_O 326 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 324 -#define _CALL_KW_NON_PY 325 -#define _CALL_LEN 326 -#define _CALL_LIST_APPEND 327 -#define _CALL_METHOD_DESCRIPTOR_FAST 328 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 329 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 330 -#define _CALL_METHOD_DESCRIPTOR_O 331 -#define _CALL_NON_PY_GENERAL 332 -#define _CALL_STR_1 333 -#define _CALL_TUPLE_1 334 -#define _CALL_TYPE_1 335 -#define _CHECK_AND_ALLOCATE_OBJECT 336 -#define _CHECK_ATTR_CLASS 337 -#define _CHECK_ATTR_METHOD_LAZY_DICT 338 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 339 +#define _CALL_ISINSTANCE 327 +#define _CALL_KW_NON_PY 328 +#define _CALL_LEN 329 +#define _CALL_LIST_APPEND 330 +#define _CALL_METHOD_DESCRIPTOR_FAST 331 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 +#define _CALL_METHOD_DESCRIPTOR_O 334 +#define _CALL_NON_PY_GENERAL 335 +#define _CALL_STR_1 336 +#define _CALL_TUPLE_1 337 +#define _CALL_TYPE_1 338 +#define _CHECK_AND_ALLOCATE_OBJECT 339 +#define _CHECK_ATTR_CLASS 340 +#define _CHECK_ATTR_METHOD_LAZY_DICT 341 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 340 -#define _CHECK_FUNCTION_EXACT_ARGS 341 -#define _CHECK_FUNCTION_VERSION 342 -#define _CHECK_FUNCTION_VERSION_INLINE 343 -#define _CHECK_FUNCTION_VERSION_KW 344 -#define _CHECK_IS_NOT_PY_CALLABLE 345 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 346 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 347 -#define _CHECK_METHOD_VERSION 348 -#define _CHECK_METHOD_VERSION_KW 349 -#define _CHECK_PEP_523 350 -#define _CHECK_PERIODIC 351 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 352 -#define _CHECK_RECURSION_REMAINING 353 -#define _CHECK_STACK_SPACE 354 -#define _CHECK_STACK_SPACE_OPERAND 355 -#define _CHECK_VALIDITY 356 -#define _COMPARE_OP 357 -#define _COMPARE_OP_FLOAT 358 -#define _COMPARE_OP_INT 359 -#define _COMPARE_OP_STR 360 -#define _CONTAINS_OP 361 -#define _CONTAINS_OP_DICT 362 -#define _CONTAINS_OP_SET 363 +#define _CHECK_FUNCTION 343 +#define _CHECK_FUNCTION_EXACT_ARGS 344 +#define _CHECK_FUNCTION_VERSION 345 +#define _CHECK_FUNCTION_VERSION_INLINE 346 +#define _CHECK_FUNCTION_VERSION_KW 347 +#define _CHECK_IS_NOT_PY_CALLABLE 348 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 +#define _CHECK_METHOD_VERSION 351 +#define _CHECK_METHOD_VERSION_KW 352 +#define _CHECK_PEP_523 353 +#define _CHECK_PERIODIC 354 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 +#define _CHECK_RECURSION_REMAINING 356 +#define _CHECK_STACK_SPACE 357 +#define _CHECK_STACK_SPACE_OPERAND 358 +#define _CHECK_VALIDITY 359 +#define _COMPARE_OP 360 +#define _COMPARE_OP_FLOAT 361 +#define _COMPARE_OP_INT 362 +#define _COMPARE_OP_STR 363 +#define _CONTAINS_OP 364 +#define _CONTAINS_OP_DICT 365 +#define _CONTAINS_OP_SET 366 #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 364 +#define _CREATE_INIT_FRAME 367 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 365 +#define _DEOPT 368 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 366 -#define _DO_CALL_FUNCTION_EX 367 -#define _DO_CALL_KW 368 +#define _DO_CALL 369 +#define _DO_CALL_FUNCTION_EX 370 +#define _DO_CALL_KW 371 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 369 +#define _ERROR_POP_N 372 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 370 -#define _EXPAND_METHOD_KW 371 -#define _FATAL_ERROR 372 +#define _EXPAND_METHOD 373 +#define _EXPAND_METHOD_KW 374 +#define _FATAL_ERROR 375 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 373 -#define _FOR_ITER_GEN_FRAME 374 -#define _FOR_ITER_TIER_TWO 375 +#define _FOR_ITER 376 +#define _FOR_ITER_GEN_FRAME 377 +#define _FOR_ITER_TIER_TWO 378 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 376 -#define _GUARD_CALLABLE_ISINSTANCE 377 -#define _GUARD_CALLABLE_LEN 378 -#define _GUARD_CALLABLE_LIST_APPEND 379 -#define _GUARD_CALLABLE_STR_1 380 -#define _GUARD_CALLABLE_TUPLE_1 381 -#define _GUARD_CALLABLE_TYPE_1 382 -#define _GUARD_DORV_NO_DICT 383 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 384 -#define _GUARD_GLOBALS_VERSION 385 -#define _GUARD_IS_FALSE_POP 386 -#define _GUARD_IS_NONE_POP 387 -#define _GUARD_IS_NOT_NONE_POP 388 -#define _GUARD_IS_TRUE_POP 389 -#define _GUARD_KEYS_VERSION 390 -#define _GUARD_NOS_DICT 391 -#define _GUARD_NOS_FLOAT 392 -#define _GUARD_NOS_INT 393 -#define _GUARD_NOS_LIST 394 -#define _GUARD_NOS_NOT_NULL 395 -#define _GUARD_NOS_NULL 396 -#define _GUARD_NOS_TUPLE 397 -#define _GUARD_NOS_UNICODE 398 -#define _GUARD_NOT_EXHAUSTED_LIST 399 -#define _GUARD_NOT_EXHAUSTED_RANGE 400 -#define _GUARD_NOT_EXHAUSTED_TUPLE 401 -#define _GUARD_THIRD_NULL 402 -#define _GUARD_TOS_ANY_SET 403 -#define _GUARD_TOS_DICT 404 -#define _GUARD_TOS_FLOAT 405 -#define _GUARD_TOS_INT 406 -#define _GUARD_TOS_LIST 407 -#define _GUARD_TOS_SLICE 408 -#define _GUARD_TOS_TUPLE 409 -#define _GUARD_TOS_UNICODE 410 -#define _GUARD_TYPE_VERSION 411 -#define _GUARD_TYPE_VERSION_AND_LOCK 412 +#define _GUARD_BINARY_OP_EXTEND 379 +#define _GUARD_CALLABLE_ISINSTANCE 380 +#define _GUARD_CALLABLE_LEN 381 +#define _GUARD_CALLABLE_LIST_APPEND 382 +#define _GUARD_CALLABLE_STR_1 383 +#define _GUARD_CALLABLE_TUPLE_1 384 +#define _GUARD_CALLABLE_TYPE_1 385 +#define _GUARD_DORV_NO_DICT 386 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 387 +#define _GUARD_GLOBALS_VERSION 388 +#define _GUARD_IS_FALSE_POP 389 +#define _GUARD_IS_NONE_POP 390 +#define _GUARD_IS_NOT_NONE_POP 391 +#define _GUARD_IS_TRUE_POP 392 +#define _GUARD_KEYS_VERSION 393 +#define _GUARD_NOS_DICT 394 +#define _GUARD_NOS_FLOAT 395 +#define _GUARD_NOS_INT 396 +#define _GUARD_NOS_LIST 397 +#define _GUARD_NOS_NOT_NULL 398 +#define _GUARD_NOS_NULL 399 +#define _GUARD_NOS_TUPLE 400 +#define _GUARD_NOS_UNICODE 401 +#define _GUARD_NOT_EXHAUSTED_LIST 402 +#define _GUARD_NOT_EXHAUSTED_RANGE 403 +#define _GUARD_NOT_EXHAUSTED_TUPLE 404 +#define _GUARD_THIRD_NULL 405 +#define _GUARD_TOS_ANY_SET 406 +#define _GUARD_TOS_DICT 407 +#define _GUARD_TOS_FLOAT 408 +#define _GUARD_TOS_INT 409 +#define _GUARD_TOS_LIST 410 +#define _GUARD_TOS_SLICE 411 +#define _GUARD_TOS_TUPLE 412 +#define _GUARD_TOS_UNICODE 413 +#define _GUARD_TYPE_VERSION 414 +#define _GUARD_TYPE_VERSION_AND_LOCK 415 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 413 -#define _INIT_CALL_PY_EXACT_ARGS 414 -#define _INIT_CALL_PY_EXACT_ARGS_0 415 -#define _INIT_CALL_PY_EXACT_ARGS_1 416 -#define _INIT_CALL_PY_EXACT_ARGS_2 417 -#define _INIT_CALL_PY_EXACT_ARGS_3 418 -#define _INIT_CALL_PY_EXACT_ARGS_4 419 -#define _INSERT_NULL 420 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 416 +#define _INIT_CALL_PY_EXACT_ARGS 417 +#define _INIT_CALL_PY_EXACT_ARGS_0 418 +#define _INIT_CALL_PY_EXACT_ARGS_1 419 +#define _INIT_CALL_PY_EXACT_ARGS_2 420 +#define _INIT_CALL_PY_EXACT_ARGS_3 421 +#define _INIT_CALL_PY_EXACT_ARGS_4 422 +#define _INSERT_NULL 423 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -175,170 +178,170 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 421 +#define _IS_NONE 424 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 422 -#define _ITER_CHECK_RANGE 423 -#define _ITER_CHECK_TUPLE 424 -#define _ITER_JUMP_LIST 425 -#define _ITER_JUMP_RANGE 426 -#define _ITER_JUMP_TUPLE 427 -#define _ITER_NEXT_LIST 428 -#define _ITER_NEXT_LIST_TIER_TWO 429 -#define _ITER_NEXT_RANGE 430 -#define _ITER_NEXT_TUPLE 431 -#define _JUMP_TO_TOP 432 +#define _ITER_CHECK_LIST 425 +#define _ITER_CHECK_RANGE 426 +#define _ITER_CHECK_TUPLE 427 +#define _ITER_JUMP_LIST 428 +#define _ITER_JUMP_RANGE 429 +#define _ITER_JUMP_TUPLE 430 +#define _ITER_NEXT_LIST 431 +#define _ITER_NEXT_LIST_TIER_TWO 432 +#define _ITER_NEXT_RANGE 433 +#define _ITER_NEXT_TUPLE 434 +#define _JUMP_TO_TOP 435 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 433 -#define _LOAD_ATTR_CLASS 434 +#define _LOAD_ATTR 436 +#define _LOAD_ATTR_CLASS 437 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 435 -#define _LOAD_ATTR_METHOD_LAZY_DICT 436 -#define _LOAD_ATTR_METHOD_NO_DICT 437 -#define _LOAD_ATTR_METHOD_WITH_VALUES 438 -#define _LOAD_ATTR_MODULE 439 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 440 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 441 -#define _LOAD_ATTR_PROPERTY_FRAME 442 -#define _LOAD_ATTR_SLOT 443 -#define _LOAD_ATTR_WITH_HINT 444 +#define _LOAD_ATTR_INSTANCE_VALUE 438 +#define _LOAD_ATTR_METHOD_LAZY_DICT 439 +#define _LOAD_ATTR_METHOD_NO_DICT 440 +#define _LOAD_ATTR_METHOD_WITH_VALUES 441 +#define _LOAD_ATTR_MODULE 442 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 443 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 444 +#define _LOAD_ATTR_PROPERTY_FRAME 445 +#define _LOAD_ATTR_SLOT 446 +#define _LOAD_ATTR_WITH_HINT 447 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 445 +#define _LOAD_BYTECODE 448 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 446 -#define _LOAD_CONST_INLINE_BORROW 447 -#define _LOAD_CONST_UNDER_INLINE 448 -#define _LOAD_CONST_UNDER_INLINE_BORROW 449 +#define _LOAD_CONST_INLINE 449 +#define _LOAD_CONST_INLINE_BORROW 450 +#define _LOAD_CONST_UNDER_INLINE 451 +#define _LOAD_CONST_UNDER_INLINE_BORROW 452 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 450 -#define _LOAD_FAST_0 451 -#define _LOAD_FAST_1 452 -#define _LOAD_FAST_2 453 -#define _LOAD_FAST_3 454 -#define _LOAD_FAST_4 455 -#define _LOAD_FAST_5 456 -#define _LOAD_FAST_6 457 -#define _LOAD_FAST_7 458 +#define _LOAD_FAST 453 +#define _LOAD_FAST_0 454 +#define _LOAD_FAST_1 455 +#define _LOAD_FAST_2 456 +#define _LOAD_FAST_3 457 +#define _LOAD_FAST_4 458 +#define _LOAD_FAST_5 459 +#define _LOAD_FAST_6 460 +#define _LOAD_FAST_7 461 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 459 -#define _LOAD_FAST_BORROW_0 460 -#define _LOAD_FAST_BORROW_1 461 -#define _LOAD_FAST_BORROW_2 462 -#define _LOAD_FAST_BORROW_3 463 -#define _LOAD_FAST_BORROW_4 464 -#define _LOAD_FAST_BORROW_5 465 -#define _LOAD_FAST_BORROW_6 466 -#define _LOAD_FAST_BORROW_7 467 +#define _LOAD_FAST_BORROW 462 +#define _LOAD_FAST_BORROW_0 463 +#define _LOAD_FAST_BORROW_1 464 +#define _LOAD_FAST_BORROW_2 465 +#define _LOAD_FAST_BORROW_3 466 +#define _LOAD_FAST_BORROW_4 467 +#define _LOAD_FAST_BORROW_5 468 +#define _LOAD_FAST_BORROW_6 469 +#define _LOAD_FAST_BORROW_7 470 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 468 -#define _LOAD_GLOBAL_BUILTINS 469 -#define _LOAD_GLOBAL_MODULE 470 +#define _LOAD_GLOBAL 471 +#define _LOAD_GLOBAL_BUILTINS 472 +#define _LOAD_GLOBAL_MODULE 473 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 471 -#define _LOAD_SMALL_INT_0 472 -#define _LOAD_SMALL_INT_1 473 -#define _LOAD_SMALL_INT_2 474 -#define _LOAD_SMALL_INT_3 475 -#define _LOAD_SPECIAL 476 +#define _LOAD_SMALL_INT 474 +#define _LOAD_SMALL_INT_0 475 +#define _LOAD_SMALL_INT_1 476 +#define _LOAD_SMALL_INT_2 477 +#define _LOAD_SMALL_INT_3 478 +#define _LOAD_SPECIAL 479 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 477 +#define _MAKE_CALLARGS_A_TUPLE 480 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 478 +#define _MAKE_WARM 481 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 479 -#define _MAYBE_EXPAND_METHOD_KW 480 -#define _MONITOR_CALL 481 -#define _MONITOR_CALL_KW 482 -#define _MONITOR_JUMP_BACKWARD 483 -#define _MONITOR_RESUME 484 +#define _MAYBE_EXPAND_METHOD 482 +#define _MAYBE_EXPAND_METHOD_KW 483 +#define _MONITOR_CALL 484 +#define _MONITOR_CALL_KW 485 +#define _MONITOR_JUMP_BACKWARD 486 +#define _MONITOR_RESUME 487 #define _NOP NOP -#define _POP_CALL 485 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 486 -#define _POP_CALL_ONE 487 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 488 -#define _POP_CALL_TWO 489 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 490 +#define _POP_CALL 488 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 489 +#define _POP_CALL_ONE 490 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 491 +#define _POP_CALL_TWO 492 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 493 #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 491 -#define _POP_JUMP_IF_TRUE 492 +#define _POP_JUMP_IF_FALSE 494 +#define _POP_JUMP_IF_TRUE 495 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 493 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 494 -#define _POP_TWO 495 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 496 +#define _POP_TOP_LOAD_CONST_INLINE 496 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 497 +#define _POP_TWO 498 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 499 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 497 +#define _PUSH_FRAME 500 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 498 -#define _PY_FRAME_GENERAL 499 -#define _PY_FRAME_KW 500 -#define _QUICKEN_RESUME 501 -#define _REPLACE_WITH_TRUE 502 +#define _PUSH_NULL_CONDITIONAL 501 +#define _PY_FRAME_GENERAL 502 +#define _PY_FRAME_KW 503 +#define _QUICKEN_RESUME 504 +#define _REPLACE_WITH_TRUE 505 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 503 -#define _SEND 504 -#define _SEND_GEN_FRAME 505 +#define _SAVE_RETURN_OFFSET 506 +#define _SEND 507 +#define _SEND_GEN_FRAME 508 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 506 -#define _STORE_ATTR 507 -#define _STORE_ATTR_INSTANCE_VALUE 508 -#define _STORE_ATTR_SLOT 509 -#define _STORE_ATTR_WITH_HINT 510 +#define _START_EXECUTOR 509 +#define _STORE_ATTR 510 +#define _STORE_ATTR_INSTANCE_VALUE 511 +#define _STORE_ATTR_SLOT 512 +#define _STORE_ATTR_WITH_HINT 513 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 511 -#define _STORE_FAST_0 512 -#define _STORE_FAST_1 513 -#define _STORE_FAST_2 514 -#define _STORE_FAST_3 515 -#define _STORE_FAST_4 516 -#define _STORE_FAST_5 517 -#define _STORE_FAST_6 518 -#define _STORE_FAST_7 519 +#define _STORE_FAST 514 +#define _STORE_FAST_0 515 +#define _STORE_FAST_1 516 +#define _STORE_FAST_2 517 +#define _STORE_FAST_3 518 +#define _STORE_FAST_4 519 +#define _STORE_FAST_5 520 +#define _STORE_FAST_6 521 +#define _STORE_FAST_7 522 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 520 -#define _STORE_SUBSCR 521 -#define _STORE_SUBSCR_DICT 522 -#define _STORE_SUBSCR_LIST_INT 523 +#define _STORE_SLICE 523 +#define _STORE_SUBSCR 524 +#define _STORE_SUBSCR_DICT 525 +#define _STORE_SUBSCR_LIST_INT 526 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 524 -#define _TO_BOOL 525 +#define _TIER2_RESUME_CHECK 527 +#define _TO_BOOL 528 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 526 +#define _TO_BOOL_LIST 529 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 527 +#define _TO_BOOL_STR 530 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 528 -#define _UNPACK_SEQUENCE_LIST 529 -#define _UNPACK_SEQUENCE_TUPLE 530 -#define _UNPACK_SEQUENCE_TWO_TUPLE 531 +#define _UNPACK_SEQUENCE 531 +#define _UNPACK_SEQUENCE_LIST 532 +#define _UNPACK_SEQUENCE_TUPLE 533 +#define _UNPACK_SEQUENCE_TWO_TUPLE 534 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 531 +#define MAX_UOP_ID 534 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 5ebe124983b862..978bc523999b36 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -92,6 +92,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -333,11 +336,13 @@ const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", @@ -347,6 +352,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SLICE] = "_BINARY_SLICE", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", @@ -782,6 +788,12 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; + case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: + return 2; + case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: + return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index cb6eae484149ee..bb8090393949a7 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -678,7 +678,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", uops) def test_float_subtract_constant_propagation(self): def testfunc(n): @@ -700,7 +700,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF", uops) def test_float_multiply_constant_propagation(self): def testfunc(n): @@ -722,7 +722,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF", uops) def test_add_unicode_propagation(self): def testfunc(n): @@ -2219,6 +2219,20 @@ def f(n): self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + def test_float_op_refcount_elimination(self): + def testfunc(args): + a, b, n = args + c = 0.0 + for _ in range(n): + c += a + b + return c + + res, ex = self._run_with_optimizer(testfunc, (0.1, 0.1, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * (0.1 + 0.1)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", uops) + def global_identity(x): return x diff --git a/Python/bytecodes.c b/Python/bytecodes.c index be22c7446f5402..0d9684da51b35c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -677,6 +677,52 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } + + pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + macro(BINARY_OP_MULTIPLY_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 40090e692e4a72..49e0cb2ebaba62 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1043,6 +1043,87 @@ break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { _PyStackRef right; _PyStackRef left; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 851e1efa0497af..02022382cb4a3a 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -342,6 +342,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal #define sym_new_truthiness _Py_uop_sym_new_truthiness +#define sym_set_skip_refcount _Py_uop_sym_set_skip_refcount +#define sym_set_dont_skip_refcount _Py_uop_sym_set_dont_skip_refcount +#define sym_is_skip_refcount _Py_uop_sym_is_skip_refcount static int optimize_to_bool( @@ -440,6 +443,13 @@ get_code_with_logging(_PyUOpInstruction *op) return co; } +// TODO (gh-134584) generate most of this table automatically +const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { + [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, + [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, + [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, +}; + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 49c6bfb6c1b01a..aa90d6b7645259 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -80,20 +80,24 @@ dummy_func(void) { if (sym_is_null(value)) { ctx->done = true; } + sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); + sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST_BORROW, (-- value)) { value = GETLOCAL(oparg); + sym_set_skip_refcount(ctx, value); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + sym_set_dont_skip_refcount(ctx, value); } op(_STORE_FAST, (value --)) { @@ -296,6 +300,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Move this to the optimizer generator. + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { @@ -316,6 +324,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Move this to the optimizer generator. + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { @@ -336,6 +348,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Move this to the optimizer generator. + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index bf7ac72d4579e7..75bfcaf30b994a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -31,6 +31,7 @@ if (sym_is_null(value)) { ctx->done = true; } + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -40,6 +41,7 @@ case _LOAD_FAST: { JitOptSymbol *value; value = GETLOCAL(oparg); + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -49,6 +51,7 @@ case _LOAD_FAST_BORROW: { JitOptSymbol *value; value = GETLOCAL(oparg); + sym_set_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -60,6 +63,7 @@ value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -434,6 +438,9 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } @@ -463,6 +470,9 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } @@ -492,10 +502,40 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { JitOptSymbol *right; JitOptSymbol *left; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 25de5d83166f64..68f1ecd1d316a4 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -99,6 +99,7 @@ sym_new(JitOptContext *ctx) } ctx->t_arena.ty_curr_number++; self->tag = JIT_SYM_UNKNOWN_TAG; + self->skip_refcount = DONT_SKIP_REFCOUNT; return self; } @@ -106,6 +107,9 @@ static void make_const(JitOptSymbol *sym, PyObject *val) { sym->tag = JIT_SYM_KNOWN_VALUE_TAG; sym->value.value = Py_NewRef(val); + // Constants don't need to be refcounted, as they are always + // kept alive by co_consts. + sym->skip_refcount = SKIP_REFCOUNT; } static inline void @@ -381,6 +385,25 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } } +void +_Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +{ + sym->skip_refcount = SKIP_REFCOUNT; +} + +void +_Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +{ + sym->skip_refcount = DONT_SKIP_REFCOUNT; +} + +bool +_Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +{ + assert(sym->skip_refcount == SKIP_REFCOUNT || sym->skip_refcount == DONT_SKIP_REFCOUNT); + return sym->skip_refcount == SKIP_REFCOUNT; +} + JitOptSymbol * _Py_uop_sym_new_unknown(JitOptContext *ctx) @@ -709,7 +732,7 @@ _Py_uop_abstractcontext_fini(JitOptContext *ctx) void _Py_uop_abstractcontext_init(JitOptContext *ctx) { - static_assert(sizeof(JitOptSymbol) <= 2 * sizeof(uint64_t), "JitOptSymbol has grown"); + static_assert(sizeof(JitOptSymbol) <= 3 * sizeof(uint64_t), "JitOptSymbol has grown"); ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. @@ -771,6 +794,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } + TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "top is refcounted"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); @@ -819,6 +843,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "42 isn't refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); @@ -847,8 +872,16 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "type should be refcounted"); + _Py_uop_sym_set_skip_refcount(ctx, sym); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "type should not be refcounted"); sym = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "None should not be refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); sym = _Py_uop_sym_new_const(ctx, Py_False); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); From ad03b1eeda291ac7f0ca414c2703de714c46fcce Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 23 May 2025 22:06:07 +0800 Subject: [PATCH 02/24] Add for common list ops --- Include/internal/pycore_uop_ids.h | 442 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 8 + Python/bytecodes.c | 54 +++ Python/executor_cases.c.h | 83 +++++ Python/optimizer_analysis.c | 2 + Python/optimizer_bytecodes.c | 17 + Python/optimizer_cases.c.h | 29 ++ 7 files changed, 415 insertions(+), 220 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index e48da341b63f1a..a8e8cce09a458e 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -25,13 +25,14 @@ extern "C" { #define _BINARY_OP_SUBSCR_DICT 313 #define _BINARY_OP_SUBSCR_INIT_CALL 314 #define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_SLICE 316 -#define _BINARY_OP_SUBSCR_STR_INT 317 -#define _BINARY_OP_SUBSCR_TUPLE_INT 318 -#define _BINARY_OP_SUBTRACT_FLOAT 319 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF 320 -#define _BINARY_OP_SUBTRACT_INT 321 -#define _BINARY_SLICE 322 +#define _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF 316 +#define _BINARY_OP_SUBSCR_LIST_SLICE 317 +#define _BINARY_OP_SUBSCR_STR_INT 318 +#define _BINARY_OP_SUBSCR_TUPLE_INT 319 +#define _BINARY_OP_SUBTRACT_FLOAT 320 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF 321 +#define _BINARY_OP_SUBTRACT_INT 322 +#define _BINARY_SLICE 323 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -40,135 +41,135 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 323 -#define _CALL_BUILTIN_FAST 324 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 -#define _CALL_BUILTIN_O 326 +#define _CALL_BUILTIN_CLASS 324 +#define _CALL_BUILTIN_FAST 325 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 326 +#define _CALL_BUILTIN_O 327 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 327 -#define _CALL_KW_NON_PY 328 -#define _CALL_LEN 329 -#define _CALL_LIST_APPEND 330 -#define _CALL_METHOD_DESCRIPTOR_FAST 331 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 -#define _CALL_METHOD_DESCRIPTOR_O 334 -#define _CALL_NON_PY_GENERAL 335 -#define _CALL_STR_1 336 -#define _CALL_TUPLE_1 337 -#define _CALL_TYPE_1 338 -#define _CHECK_AND_ALLOCATE_OBJECT 339 -#define _CHECK_ATTR_CLASS 340 -#define _CHECK_ATTR_METHOD_LAZY_DICT 341 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 +#define _CALL_ISINSTANCE 328 +#define _CALL_KW_NON_PY 329 +#define _CALL_LEN 330 +#define _CALL_LIST_APPEND 331 +#define _CALL_METHOD_DESCRIPTOR_FAST 332 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 333 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 334 +#define _CALL_METHOD_DESCRIPTOR_O 335 +#define _CALL_NON_PY_GENERAL 336 +#define _CALL_STR_1 337 +#define _CALL_TUPLE_1 338 +#define _CALL_TYPE_1 339 +#define _CHECK_AND_ALLOCATE_OBJECT 340 +#define _CHECK_ATTR_CLASS 341 +#define _CHECK_ATTR_METHOD_LAZY_DICT 342 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 343 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 343 -#define _CHECK_FUNCTION_EXACT_ARGS 344 -#define _CHECK_FUNCTION_VERSION 345 -#define _CHECK_FUNCTION_VERSION_INLINE 346 -#define _CHECK_FUNCTION_VERSION_KW 347 -#define _CHECK_IS_NOT_PY_CALLABLE 348 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 -#define _CHECK_METHOD_VERSION 351 -#define _CHECK_METHOD_VERSION_KW 352 -#define _CHECK_PEP_523 353 -#define _CHECK_PERIODIC 354 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 -#define _CHECK_RECURSION_REMAINING 356 -#define _CHECK_STACK_SPACE 357 -#define _CHECK_STACK_SPACE_OPERAND 358 -#define _CHECK_VALIDITY 359 -#define _COMPARE_OP 360 -#define _COMPARE_OP_FLOAT 361 -#define _COMPARE_OP_INT 362 -#define _COMPARE_OP_STR 363 -#define _CONTAINS_OP 364 -#define _CONTAINS_OP_DICT 365 -#define _CONTAINS_OP_SET 366 +#define _CHECK_FUNCTION 344 +#define _CHECK_FUNCTION_EXACT_ARGS 345 +#define _CHECK_FUNCTION_VERSION 346 +#define _CHECK_FUNCTION_VERSION_INLINE 347 +#define _CHECK_FUNCTION_VERSION_KW 348 +#define _CHECK_IS_NOT_PY_CALLABLE 349 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 350 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 351 +#define _CHECK_METHOD_VERSION 352 +#define _CHECK_METHOD_VERSION_KW 353 +#define _CHECK_PEP_523 354 +#define _CHECK_PERIODIC 355 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 356 +#define _CHECK_RECURSION_REMAINING 357 +#define _CHECK_STACK_SPACE 358 +#define _CHECK_STACK_SPACE_OPERAND 359 +#define _CHECK_VALIDITY 360 +#define _COMPARE_OP 361 +#define _COMPARE_OP_FLOAT 362 +#define _COMPARE_OP_INT 363 +#define _COMPARE_OP_STR 364 +#define _CONTAINS_OP 365 +#define _CONTAINS_OP_DICT 366 +#define _CONTAINS_OP_SET 367 #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 367 +#define _CREATE_INIT_FRAME 368 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 368 +#define _DEOPT 369 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 369 -#define _DO_CALL_FUNCTION_EX 370 -#define _DO_CALL_KW 371 +#define _DO_CALL 370 +#define _DO_CALL_FUNCTION_EX 371 +#define _DO_CALL_KW 372 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 372 +#define _ERROR_POP_N 373 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 373 -#define _EXPAND_METHOD_KW 374 -#define _FATAL_ERROR 375 +#define _EXPAND_METHOD 374 +#define _EXPAND_METHOD_KW 375 +#define _FATAL_ERROR 376 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 376 -#define _FOR_ITER_GEN_FRAME 377 -#define _FOR_ITER_TIER_TWO 378 +#define _FOR_ITER 377 +#define _FOR_ITER_GEN_FRAME 378 +#define _FOR_ITER_TIER_TWO 379 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 379 -#define _GUARD_CALLABLE_ISINSTANCE 380 -#define _GUARD_CALLABLE_LEN 381 -#define _GUARD_CALLABLE_LIST_APPEND 382 -#define _GUARD_CALLABLE_STR_1 383 -#define _GUARD_CALLABLE_TUPLE_1 384 -#define _GUARD_CALLABLE_TYPE_1 385 -#define _GUARD_DORV_NO_DICT 386 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 387 -#define _GUARD_GLOBALS_VERSION 388 -#define _GUARD_IS_FALSE_POP 389 -#define _GUARD_IS_NONE_POP 390 -#define _GUARD_IS_NOT_NONE_POP 391 -#define _GUARD_IS_TRUE_POP 392 -#define _GUARD_KEYS_VERSION 393 -#define _GUARD_NOS_DICT 394 -#define _GUARD_NOS_FLOAT 395 -#define _GUARD_NOS_INT 396 -#define _GUARD_NOS_LIST 397 -#define _GUARD_NOS_NOT_NULL 398 -#define _GUARD_NOS_NULL 399 -#define _GUARD_NOS_TUPLE 400 -#define _GUARD_NOS_UNICODE 401 -#define _GUARD_NOT_EXHAUSTED_LIST 402 -#define _GUARD_NOT_EXHAUSTED_RANGE 403 -#define _GUARD_NOT_EXHAUSTED_TUPLE 404 -#define _GUARD_THIRD_NULL 405 -#define _GUARD_TOS_ANY_SET 406 -#define _GUARD_TOS_DICT 407 -#define _GUARD_TOS_FLOAT 408 -#define _GUARD_TOS_INT 409 -#define _GUARD_TOS_LIST 410 -#define _GUARD_TOS_SLICE 411 -#define _GUARD_TOS_TUPLE 412 -#define _GUARD_TOS_UNICODE 413 -#define _GUARD_TYPE_VERSION 414 -#define _GUARD_TYPE_VERSION_AND_LOCK 415 +#define _GUARD_BINARY_OP_EXTEND 380 +#define _GUARD_CALLABLE_ISINSTANCE 381 +#define _GUARD_CALLABLE_LEN 382 +#define _GUARD_CALLABLE_LIST_APPEND 383 +#define _GUARD_CALLABLE_STR_1 384 +#define _GUARD_CALLABLE_TUPLE_1 385 +#define _GUARD_CALLABLE_TYPE_1 386 +#define _GUARD_DORV_NO_DICT 387 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 388 +#define _GUARD_GLOBALS_VERSION 389 +#define _GUARD_IS_FALSE_POP 390 +#define _GUARD_IS_NONE_POP 391 +#define _GUARD_IS_NOT_NONE_POP 392 +#define _GUARD_IS_TRUE_POP 393 +#define _GUARD_KEYS_VERSION 394 +#define _GUARD_NOS_DICT 395 +#define _GUARD_NOS_FLOAT 396 +#define _GUARD_NOS_INT 397 +#define _GUARD_NOS_LIST 398 +#define _GUARD_NOS_NOT_NULL 399 +#define _GUARD_NOS_NULL 400 +#define _GUARD_NOS_TUPLE 401 +#define _GUARD_NOS_UNICODE 402 +#define _GUARD_NOT_EXHAUSTED_LIST 403 +#define _GUARD_NOT_EXHAUSTED_RANGE 404 +#define _GUARD_NOT_EXHAUSTED_TUPLE 405 +#define _GUARD_THIRD_NULL 406 +#define _GUARD_TOS_ANY_SET 407 +#define _GUARD_TOS_DICT 408 +#define _GUARD_TOS_FLOAT 409 +#define _GUARD_TOS_INT 410 +#define _GUARD_TOS_LIST 411 +#define _GUARD_TOS_SLICE 412 +#define _GUARD_TOS_TUPLE 413 +#define _GUARD_TOS_UNICODE 414 +#define _GUARD_TYPE_VERSION 415 +#define _GUARD_TYPE_VERSION_AND_LOCK 416 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 416 -#define _INIT_CALL_PY_EXACT_ARGS 417 -#define _INIT_CALL_PY_EXACT_ARGS_0 418 -#define _INIT_CALL_PY_EXACT_ARGS_1 419 -#define _INIT_CALL_PY_EXACT_ARGS_2 420 -#define _INIT_CALL_PY_EXACT_ARGS_3 421 -#define _INIT_CALL_PY_EXACT_ARGS_4 422 -#define _INSERT_NULL 423 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 417 +#define _INIT_CALL_PY_EXACT_ARGS 418 +#define _INIT_CALL_PY_EXACT_ARGS_0 419 +#define _INIT_CALL_PY_EXACT_ARGS_1 420 +#define _INIT_CALL_PY_EXACT_ARGS_2 421 +#define _INIT_CALL_PY_EXACT_ARGS_3 422 +#define _INIT_CALL_PY_EXACT_ARGS_4 423 +#define _INSERT_NULL 424 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -178,170 +179,171 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 424 +#define _IS_NONE 425 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 425 -#define _ITER_CHECK_RANGE 426 -#define _ITER_CHECK_TUPLE 427 -#define _ITER_JUMP_LIST 428 -#define _ITER_JUMP_RANGE 429 -#define _ITER_JUMP_TUPLE 430 -#define _ITER_NEXT_LIST 431 -#define _ITER_NEXT_LIST_TIER_TWO 432 -#define _ITER_NEXT_RANGE 433 -#define _ITER_NEXT_TUPLE 434 -#define _JUMP_TO_TOP 435 +#define _ITER_CHECK_LIST 426 +#define _ITER_CHECK_RANGE 427 +#define _ITER_CHECK_TUPLE 428 +#define _ITER_JUMP_LIST 429 +#define _ITER_JUMP_RANGE 430 +#define _ITER_JUMP_TUPLE 431 +#define _ITER_NEXT_LIST 432 +#define _ITER_NEXT_LIST_TIER_TWO 433 +#define _ITER_NEXT_RANGE 434 +#define _ITER_NEXT_TUPLE 435 +#define _JUMP_TO_TOP 436 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 436 -#define _LOAD_ATTR_CLASS 437 +#define _LOAD_ATTR 437 +#define _LOAD_ATTR_CLASS 438 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 438 -#define _LOAD_ATTR_METHOD_LAZY_DICT 439 -#define _LOAD_ATTR_METHOD_NO_DICT 440 -#define _LOAD_ATTR_METHOD_WITH_VALUES 441 -#define _LOAD_ATTR_MODULE 442 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 443 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 444 -#define _LOAD_ATTR_PROPERTY_FRAME 445 -#define _LOAD_ATTR_SLOT 446 -#define _LOAD_ATTR_WITH_HINT 447 +#define _LOAD_ATTR_INSTANCE_VALUE 439 +#define _LOAD_ATTR_METHOD_LAZY_DICT 440 +#define _LOAD_ATTR_METHOD_NO_DICT 441 +#define _LOAD_ATTR_METHOD_WITH_VALUES 442 +#define _LOAD_ATTR_MODULE 443 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 444 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 445 +#define _LOAD_ATTR_PROPERTY_FRAME 446 +#define _LOAD_ATTR_SLOT 447 +#define _LOAD_ATTR_WITH_HINT 448 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 448 +#define _LOAD_BYTECODE 449 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 449 -#define _LOAD_CONST_INLINE_BORROW 450 -#define _LOAD_CONST_UNDER_INLINE 451 -#define _LOAD_CONST_UNDER_INLINE_BORROW 452 +#define _LOAD_CONST_INLINE 450 +#define _LOAD_CONST_INLINE_BORROW 451 +#define _LOAD_CONST_UNDER_INLINE 452 +#define _LOAD_CONST_UNDER_INLINE_BORROW 453 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 453 -#define _LOAD_FAST_0 454 -#define _LOAD_FAST_1 455 -#define _LOAD_FAST_2 456 -#define _LOAD_FAST_3 457 -#define _LOAD_FAST_4 458 -#define _LOAD_FAST_5 459 -#define _LOAD_FAST_6 460 -#define _LOAD_FAST_7 461 +#define _LOAD_FAST 454 +#define _LOAD_FAST_0 455 +#define _LOAD_FAST_1 456 +#define _LOAD_FAST_2 457 +#define _LOAD_FAST_3 458 +#define _LOAD_FAST_4 459 +#define _LOAD_FAST_5 460 +#define _LOAD_FAST_6 461 +#define _LOAD_FAST_7 462 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 462 -#define _LOAD_FAST_BORROW_0 463 -#define _LOAD_FAST_BORROW_1 464 -#define _LOAD_FAST_BORROW_2 465 -#define _LOAD_FAST_BORROW_3 466 -#define _LOAD_FAST_BORROW_4 467 -#define _LOAD_FAST_BORROW_5 468 -#define _LOAD_FAST_BORROW_6 469 -#define _LOAD_FAST_BORROW_7 470 +#define _LOAD_FAST_BORROW 463 +#define _LOAD_FAST_BORROW_0 464 +#define _LOAD_FAST_BORROW_1 465 +#define _LOAD_FAST_BORROW_2 466 +#define _LOAD_FAST_BORROW_3 467 +#define _LOAD_FAST_BORROW_4 468 +#define _LOAD_FAST_BORROW_5 469 +#define _LOAD_FAST_BORROW_6 470 +#define _LOAD_FAST_BORROW_7 471 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 471 -#define _LOAD_GLOBAL_BUILTINS 472 -#define _LOAD_GLOBAL_MODULE 473 +#define _LOAD_GLOBAL 472 +#define _LOAD_GLOBAL_BUILTINS 473 +#define _LOAD_GLOBAL_MODULE 474 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 474 -#define _LOAD_SMALL_INT_0 475 -#define _LOAD_SMALL_INT_1 476 -#define _LOAD_SMALL_INT_2 477 -#define _LOAD_SMALL_INT_3 478 -#define _LOAD_SPECIAL 479 +#define _LOAD_SMALL_INT 475 +#define _LOAD_SMALL_INT_0 476 +#define _LOAD_SMALL_INT_1 477 +#define _LOAD_SMALL_INT_2 478 +#define _LOAD_SMALL_INT_3 479 +#define _LOAD_SPECIAL 480 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 480 +#define _MAKE_CALLARGS_A_TUPLE 481 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 481 +#define _MAKE_WARM 482 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 482 -#define _MAYBE_EXPAND_METHOD_KW 483 -#define _MONITOR_CALL 484 -#define _MONITOR_CALL_KW 485 -#define _MONITOR_JUMP_BACKWARD 486 -#define _MONITOR_RESUME 487 +#define _MAYBE_EXPAND_METHOD 483 +#define _MAYBE_EXPAND_METHOD_KW 484 +#define _MONITOR_CALL 485 +#define _MONITOR_CALL_KW 486 +#define _MONITOR_JUMP_BACKWARD 487 +#define _MONITOR_RESUME 488 #define _NOP NOP -#define _POP_CALL 488 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 489 -#define _POP_CALL_ONE 490 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 491 -#define _POP_CALL_TWO 492 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 493 +#define _POP_CALL 489 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 490 +#define _POP_CALL_ONE 491 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 492 +#define _POP_CALL_TWO 493 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 494 #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 494 -#define _POP_JUMP_IF_TRUE 495 +#define _POP_JUMP_IF_FALSE 495 +#define _POP_JUMP_IF_TRUE 496 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 496 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 497 -#define _POP_TWO 498 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 499 +#define _POP_TOP_LOAD_CONST_INLINE 497 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 498 +#define _POP_TWO 499 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 500 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 500 +#define _PUSH_FRAME 501 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 501 -#define _PY_FRAME_GENERAL 502 -#define _PY_FRAME_KW 503 -#define _QUICKEN_RESUME 504 -#define _REPLACE_WITH_TRUE 505 +#define _PUSH_NULL_CONDITIONAL 502 +#define _PY_FRAME_GENERAL 503 +#define _PY_FRAME_KW 504 +#define _QUICKEN_RESUME 505 +#define _REPLACE_WITH_TRUE 506 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 506 -#define _SEND 507 -#define _SEND_GEN_FRAME 508 +#define _SAVE_RETURN_OFFSET 507 +#define _SEND 508 +#define _SEND_GEN_FRAME 509 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 509 -#define _STORE_ATTR 510 -#define _STORE_ATTR_INSTANCE_VALUE 511 -#define _STORE_ATTR_SLOT 512 -#define _STORE_ATTR_WITH_HINT 513 +#define _START_EXECUTOR 510 +#define _STORE_ATTR 511 +#define _STORE_ATTR_INSTANCE_VALUE 512 +#define _STORE_ATTR_SLOT 513 +#define _STORE_ATTR_WITH_HINT 514 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 514 -#define _STORE_FAST_0 515 -#define _STORE_FAST_1 516 -#define _STORE_FAST_2 517 -#define _STORE_FAST_3 518 -#define _STORE_FAST_4 519 -#define _STORE_FAST_5 520 -#define _STORE_FAST_6 521 -#define _STORE_FAST_7 522 +#define _STORE_FAST 515 +#define _STORE_FAST_0 516 +#define _STORE_FAST_1 517 +#define _STORE_FAST_2 518 +#define _STORE_FAST_3 519 +#define _STORE_FAST_4 520 +#define _STORE_FAST_5 521 +#define _STORE_FAST_6 522 +#define _STORE_FAST_7 523 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 523 -#define _STORE_SUBSCR 524 -#define _STORE_SUBSCR_DICT 525 -#define _STORE_SUBSCR_LIST_INT 526 +#define _STORE_SLICE 524 +#define _STORE_SUBSCR 525 +#define _STORE_SUBSCR_DICT 526 +#define _STORE_SUBSCR_LIST_INT 527 +#define _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF 528 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 527 -#define _TO_BOOL 528 +#define _TIER2_RESUME_CHECK 529 +#define _TO_BOOL 530 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 529 +#define _TO_BOOL_LIST 531 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 530 +#define _TO_BOOL_STR 532 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 531 -#define _UNPACK_SEQUENCE_LIST 532 -#define _UNPACK_SEQUENCE_TUPLE 533 -#define _UNPACK_SEQUENCE_TWO_TUPLE 534 +#define _UNPACK_SEQUENCE 533 +#define _UNPACK_SEQUENCE_LIST 534 +#define _UNPACK_SEQUENCE_TUPLE 535 +#define _UNPACK_SEQUENCE_TWO_TUPLE 536 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 534 +#define MAX_UOP_ID 536 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 978bc523999b36..1efc90238132d8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -102,6 +102,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, @@ -116,6 +117,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -348,6 +350,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", + [_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF] = "_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", @@ -620,6 +623,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR] = "_STORE_SUBSCR", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF] = "_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF", [_SWAP] = "_SWAP", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", @@ -808,6 +812,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 4; case _BINARY_OP_SUBSCR_LIST_INT: return 2; + case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: + return 2; case _BINARY_OP_SUBSCR_LIST_SLICE: return 2; case _BINARY_OP_SUBSCR_STR_INT: @@ -836,6 +842,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_LIST_INT: return 3; + case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: + return 3; case _STORE_SUBSCR_DICT: return 3; case _DELETE_SUBSCR: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0d9684da51b35c..adcf95d7b763bd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -907,6 +907,32 @@ dummy_func( DECREF_INPUTS(); } + op(_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF, (list_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; +#ifdef Py_GIL_DISABLED + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + DEOPT_IF(res_o == NULL); + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); +#else + DEOPT_IF(index >= PyList_GET_SIZE(list)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); +#endif + STAT_INC(BINARY_OP, hit); + INPUTS_DEAD(); + } + macro(BINARY_OP_SUBSCR_LIST_SLICE) = _GUARD_TOS_SLICE + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_SLICE; @@ -1109,6 +1135,34 @@ dummy_func( Py_DECREF(old_value); } + op(_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF, (value, list_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list)); + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true); + } + STAT_INC(STORE_SUBSCR, hit); + + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); // unlock before decrefs! + DEAD(sub_st); + DEAD(list_st); + Py_DECREF(old_value); + } + macro(STORE_SUBSCR_DICT) = _GUARD_NOS_DICT + unused/1 + _STORE_SUBSCR_DICT; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 49e0cb2ebaba62..d536fd68c6095b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1384,6 +1384,48 @@ break; } + case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_SUBSCR_LIST_SLICE: { _PyStackRef sub_st; _PyStackRef list_st; @@ -1743,6 +1785,47 @@ break; } + case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef value; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } + case _STORE_SUBSCR_DICT: { _PyStackRef sub; _PyStackRef dict_st; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 02022382cb4a3a..7acbf4672136e1 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -448,6 +448,8 @@ const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, + [_STORE_SUBSCR_LIST_INT] = _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF, + [_BINARY_OP_SUBSCR_LIST_INT] = _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF, }; /* 1 for success, 0 for not ready, cannot error at the moment. */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index aa90d6b7645259..3f72d73abcfb52 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -104,6 +104,15 @@ dummy_func(void) { GETLOCAL(oparg) = value; } + op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) { + // Can't move this to the optimizer generator for now, as it requires list_st and sub_st + // to be borrowed, but not value. + // Alternatively, we could just stricten it and require all to be borrowed. + if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } + } + op(_PUSH_NULL, (-- res)) { res = sym_new_null(ctx); } @@ -394,6 +403,14 @@ dummy_func(void) { ctx->done = true; } + op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) { + // TODO (gh-134584): Move this to the optimizer generator. + if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } + res = sym_new_not_null(ctx); + } + op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { res = sym_new_type(ctx, &PyUnicode_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 75bfcaf30b994a..86412bf848623f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -617,6 +617,22 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { + JitOptSymbol *sub_st; + JitOptSymbol *list_st; + JitOptSymbol *res; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -759,6 +775,19 @@ } case _STORE_SUBSCR_LIST_INT: { + JitOptSymbol *sub_st; + JitOptSymbol *list_st; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: { stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; From 7f90d0cf7357b2b703110c685311fbac5f317ac1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 23 May 2025 22:09:28 +0800 Subject: [PATCH 03/24] Fix test, rename --- Include/internal/pycore_uop_ids.h | 10 ++++----- Include/internal/pycore_uop_metadata.h | 30 +++++++++++++------------- Lib/test/test_capi/test_opt.py | 12 +++++------ Python/bytecodes.c | 10 ++++----- Python/executor_cases.c.h | 10 ++++----- Python/optimizer_analysis.c | 10 ++++----- Python/optimizer_cases.c.h | 10 ++++----- 7 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a8e8cce09a458e..d105d7e16c740e 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,24 +13,24 @@ extern "C" { #define _SET_IP 301 #define _BINARY_OP 302 #define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF 304 +#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS 304 #define _BINARY_OP_ADD_INT 305 #define _BINARY_OP_ADD_UNICODE 306 #define _BINARY_OP_EXTEND 307 #define _BINARY_OP_INPLACE_ADD_UNICODE 308 #define _BINARY_OP_MULTIPLY_FLOAT 309 -#define _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF 310 +#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS 310 #define _BINARY_OP_MULTIPLY_INT 311 #define _BINARY_OP_SUBSCR_CHECK_FUNC 312 #define _BINARY_OP_SUBSCR_DICT 313 #define _BINARY_OP_SUBSCR_INIT_CALL 314 #define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF 316 +#define _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS 316 #define _BINARY_OP_SUBSCR_LIST_SLICE 317 #define _BINARY_OP_SUBSCR_STR_INT 318 #define _BINARY_OP_SUBSCR_TUPLE_INT 319 #define _BINARY_OP_SUBTRACT_FLOAT 320 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF 321 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 321 #define _BINARY_OP_SUBTRACT_INT 322 #define _BINARY_SLICE 323 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION @@ -324,7 +324,7 @@ extern "C" { #define _STORE_SUBSCR 525 #define _STORE_SUBSCR_DICT 526 #define _STORE_SUBSCR_LIST_INT 527 -#define _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF 528 +#define _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS 528 #define _SWAP SWAP #define _TIER2_RESUME_CHECK 529 #define _TO_BOOL 530 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1efc90238132d8..dcbbdb7fc6a8fc 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -92,9 +92,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -102,7 +102,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, @@ -117,7 +117,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -338,24 +338,24 @@ const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", - [_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", - [_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF", + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", - [_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF] = "_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF", + [_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", - [_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF] = "_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF", + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SLICE] = "_BINARY_SLICE", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", @@ -623,7 +623,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR] = "_STORE_SUBSCR", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", - [_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF] = "_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF", + [_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = "_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS", [_SWAP] = "_SWAP", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", @@ -792,11 +792,11 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; - case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: return 2; - case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: return 2; - case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: return 2; case _BINARY_OP_ADD_UNICODE: return 2; @@ -812,7 +812,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 4; case _BINARY_OP_SUBSCR_LIST_INT: return 2; - case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: + case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: return 2; case _BINARY_OP_SUBSCR_LIST_SLICE: return 2; @@ -842,7 +842,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_LIST_INT: return 3; - case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: + case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: return 3; case _STORE_SUBSCR_DICT: return 3; diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index bb8090393949a7..7ccbc817bd280f 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -678,7 +678,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) def test_float_subtract_constant_propagation(self): def testfunc(n): @@ -700,7 +700,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF", uops) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", uops) def test_float_multiply_constant_propagation(self): def testfunc(n): @@ -722,7 +722,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF", uops) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", uops) def test_add_unicode_propagation(self): def testfunc(n): @@ -1707,10 +1707,10 @@ def f(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertEqual(uops.count("_GUARD_NOS_LIST"), 0) - self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT"), 1) + self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS"), 1) self.assertEqual(uops.count("_GUARD_TOS_LIST"), 0) self.assertEqual(uops.count("_UNPACK_SEQUENCE_LIST"), 1) - self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS"), 1) self.assertEqual(uops.count("_TO_BOOL_LIST"), 1) def test_remove_guard_for_known_type_set(self): @@ -2231,7 +2231,7 @@ def testfunc(args): self.assertAlmostEqual(res, TIER2_THRESHOLD * (0.1 + 0.1)) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) def global_identity(x): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index adcf95d7b763bd..77211b63ce0739 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -678,7 +678,7 @@ dummy_func( } - pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -693,7 +693,7 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } - pure op(_BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + pure op(_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -708,7 +708,7 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } - pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, (left, right -- res)) { + pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -907,7 +907,7 @@ dummy_func( DECREF_INPUTS(); } - op(_BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF, (list_st, sub_st -- res)) { + op(_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS, (list_st, sub_st -- res)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); @@ -1135,7 +1135,7 @@ dummy_func( Py_DECREF(old_value); } - op(_STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF, (value, list_st, sub_st -- )) { + op(_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS, (value, list_st, sub_st -- )) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d536fd68c6095b..aae427e4b64b96 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1043,7 +1043,7 @@ break; } - case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -1070,7 +1070,7 @@ break; } - case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -1097,7 +1097,7 @@ break; } - case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -1384,7 +1384,7 @@ break; } - case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef res; @@ -1785,7 +1785,7 @@ break; } - case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef value; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 7acbf4672136e1..6270cd5be71a94 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -445,11 +445,11 @@ get_code_with_logging(_PyUOpInstruction *op) // TODO (gh-134584) generate most of this table automatically const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { - [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF, - [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF, - [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF, - [_STORE_SUBSCR_LIST_INT] = _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF, - [_BINARY_OP_SUBSCR_LIST_INT] = _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF, + [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, + [_STORE_SUBSCR_LIST_INT] = _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBSCR_LIST_INT] = _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS, }; /* 1 for success, 0 for not ready, cannot error at the moment. */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 86412bf848623f..8419eefbf7b6e0 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -509,7 +509,7 @@ break; } - case _BINARY_OP_MULTIPLY_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -518,7 +518,7 @@ break; } - case _BINARY_OP_ADD_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -527,7 +527,7 @@ break; } - case _BINARY_OP_SUBTRACT_FLOAT__NO_INPUT_DECREF: { + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -632,7 +632,7 @@ break; } - case _BINARY_OP_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -787,7 +787,7 @@ break; } - case _STORE_SUBSCR_LIST_INT__NO_INPUT_DECREF: { + case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; From a42b43483e98039402259b6547d7fd41c76c2c7b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 23 May 2025 22:51:29 +0800 Subject: [PATCH 04/24] Remove list optimizations to minimize PR --- Include/internal/pycore_uop_ids.h | 442 ++++++++++++------------- Include/internal/pycore_uop_metadata.h | 8 - Lib/test/test_capi/test_opt.py | 4 +- Python/bytecodes.c | 54 --- Python/executor_cases.c.h | 83 ----- Python/optimizer_analysis.c | 2 - Python/optimizer_bytecodes.c | 17 - Python/optimizer_cases.c.h | 29 -- 8 files changed, 222 insertions(+), 417 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d105d7e16c740e..2ba13418a18cfc 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -25,14 +25,13 @@ extern "C" { #define _BINARY_OP_SUBSCR_DICT 313 #define _BINARY_OP_SUBSCR_INIT_CALL 314 #define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS 316 -#define _BINARY_OP_SUBSCR_LIST_SLICE 317 -#define _BINARY_OP_SUBSCR_STR_INT 318 -#define _BINARY_OP_SUBSCR_TUPLE_INT 319 -#define _BINARY_OP_SUBTRACT_FLOAT 320 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 321 -#define _BINARY_OP_SUBTRACT_INT 322 -#define _BINARY_SLICE 323 +#define _BINARY_OP_SUBSCR_LIST_SLICE 316 +#define _BINARY_OP_SUBSCR_STR_INT 317 +#define _BINARY_OP_SUBSCR_TUPLE_INT 318 +#define _BINARY_OP_SUBTRACT_FLOAT 319 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 320 +#define _BINARY_OP_SUBTRACT_INT 321 +#define _BINARY_SLICE 322 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -41,135 +40,135 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 324 -#define _CALL_BUILTIN_FAST 325 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 326 -#define _CALL_BUILTIN_O 327 +#define _CALL_BUILTIN_CLASS 323 +#define _CALL_BUILTIN_FAST 324 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 +#define _CALL_BUILTIN_O 326 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 328 -#define _CALL_KW_NON_PY 329 -#define _CALL_LEN 330 -#define _CALL_LIST_APPEND 331 -#define _CALL_METHOD_DESCRIPTOR_FAST 332 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 333 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 334 -#define _CALL_METHOD_DESCRIPTOR_O 335 -#define _CALL_NON_PY_GENERAL 336 -#define _CALL_STR_1 337 -#define _CALL_TUPLE_1 338 -#define _CALL_TYPE_1 339 -#define _CHECK_AND_ALLOCATE_OBJECT 340 -#define _CHECK_ATTR_CLASS 341 -#define _CHECK_ATTR_METHOD_LAZY_DICT 342 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 343 +#define _CALL_ISINSTANCE 327 +#define _CALL_KW_NON_PY 328 +#define _CALL_LEN 329 +#define _CALL_LIST_APPEND 330 +#define _CALL_METHOD_DESCRIPTOR_FAST 331 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 +#define _CALL_METHOD_DESCRIPTOR_O 334 +#define _CALL_NON_PY_GENERAL 335 +#define _CALL_STR_1 336 +#define _CALL_TUPLE_1 337 +#define _CALL_TYPE_1 338 +#define _CHECK_AND_ALLOCATE_OBJECT 339 +#define _CHECK_ATTR_CLASS 340 +#define _CHECK_ATTR_METHOD_LAZY_DICT 341 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 344 -#define _CHECK_FUNCTION_EXACT_ARGS 345 -#define _CHECK_FUNCTION_VERSION 346 -#define _CHECK_FUNCTION_VERSION_INLINE 347 -#define _CHECK_FUNCTION_VERSION_KW 348 -#define _CHECK_IS_NOT_PY_CALLABLE 349 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 350 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 351 -#define _CHECK_METHOD_VERSION 352 -#define _CHECK_METHOD_VERSION_KW 353 -#define _CHECK_PEP_523 354 -#define _CHECK_PERIODIC 355 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 356 -#define _CHECK_RECURSION_REMAINING 357 -#define _CHECK_STACK_SPACE 358 -#define _CHECK_STACK_SPACE_OPERAND 359 -#define _CHECK_VALIDITY 360 -#define _COMPARE_OP 361 -#define _COMPARE_OP_FLOAT 362 -#define _COMPARE_OP_INT 363 -#define _COMPARE_OP_STR 364 -#define _CONTAINS_OP 365 -#define _CONTAINS_OP_DICT 366 -#define _CONTAINS_OP_SET 367 +#define _CHECK_FUNCTION 343 +#define _CHECK_FUNCTION_EXACT_ARGS 344 +#define _CHECK_FUNCTION_VERSION 345 +#define _CHECK_FUNCTION_VERSION_INLINE 346 +#define _CHECK_FUNCTION_VERSION_KW 347 +#define _CHECK_IS_NOT_PY_CALLABLE 348 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 +#define _CHECK_METHOD_VERSION 351 +#define _CHECK_METHOD_VERSION_KW 352 +#define _CHECK_PEP_523 353 +#define _CHECK_PERIODIC 354 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 +#define _CHECK_RECURSION_REMAINING 356 +#define _CHECK_STACK_SPACE 357 +#define _CHECK_STACK_SPACE_OPERAND 358 +#define _CHECK_VALIDITY 359 +#define _COMPARE_OP 360 +#define _COMPARE_OP_FLOAT 361 +#define _COMPARE_OP_INT 362 +#define _COMPARE_OP_STR 363 +#define _CONTAINS_OP 364 +#define _CONTAINS_OP_DICT 365 +#define _CONTAINS_OP_SET 366 #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 368 +#define _CREATE_INIT_FRAME 367 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 369 +#define _DEOPT 368 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 370 -#define _DO_CALL_FUNCTION_EX 371 -#define _DO_CALL_KW 372 +#define _DO_CALL 369 +#define _DO_CALL_FUNCTION_EX 370 +#define _DO_CALL_KW 371 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 373 +#define _ERROR_POP_N 372 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 374 -#define _EXPAND_METHOD_KW 375 -#define _FATAL_ERROR 376 +#define _EXPAND_METHOD 373 +#define _EXPAND_METHOD_KW 374 +#define _FATAL_ERROR 375 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 377 -#define _FOR_ITER_GEN_FRAME 378 -#define _FOR_ITER_TIER_TWO 379 +#define _FOR_ITER 376 +#define _FOR_ITER_GEN_FRAME 377 +#define _FOR_ITER_TIER_TWO 378 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 380 -#define _GUARD_CALLABLE_ISINSTANCE 381 -#define _GUARD_CALLABLE_LEN 382 -#define _GUARD_CALLABLE_LIST_APPEND 383 -#define _GUARD_CALLABLE_STR_1 384 -#define _GUARD_CALLABLE_TUPLE_1 385 -#define _GUARD_CALLABLE_TYPE_1 386 -#define _GUARD_DORV_NO_DICT 387 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 388 -#define _GUARD_GLOBALS_VERSION 389 -#define _GUARD_IS_FALSE_POP 390 -#define _GUARD_IS_NONE_POP 391 -#define _GUARD_IS_NOT_NONE_POP 392 -#define _GUARD_IS_TRUE_POP 393 -#define _GUARD_KEYS_VERSION 394 -#define _GUARD_NOS_DICT 395 -#define _GUARD_NOS_FLOAT 396 -#define _GUARD_NOS_INT 397 -#define _GUARD_NOS_LIST 398 -#define _GUARD_NOS_NOT_NULL 399 -#define _GUARD_NOS_NULL 400 -#define _GUARD_NOS_TUPLE 401 -#define _GUARD_NOS_UNICODE 402 -#define _GUARD_NOT_EXHAUSTED_LIST 403 -#define _GUARD_NOT_EXHAUSTED_RANGE 404 -#define _GUARD_NOT_EXHAUSTED_TUPLE 405 -#define _GUARD_THIRD_NULL 406 -#define _GUARD_TOS_ANY_SET 407 -#define _GUARD_TOS_DICT 408 -#define _GUARD_TOS_FLOAT 409 -#define _GUARD_TOS_INT 410 -#define _GUARD_TOS_LIST 411 -#define _GUARD_TOS_SLICE 412 -#define _GUARD_TOS_TUPLE 413 -#define _GUARD_TOS_UNICODE 414 -#define _GUARD_TYPE_VERSION 415 -#define _GUARD_TYPE_VERSION_AND_LOCK 416 +#define _GUARD_BINARY_OP_EXTEND 379 +#define _GUARD_CALLABLE_ISINSTANCE 380 +#define _GUARD_CALLABLE_LEN 381 +#define _GUARD_CALLABLE_LIST_APPEND 382 +#define _GUARD_CALLABLE_STR_1 383 +#define _GUARD_CALLABLE_TUPLE_1 384 +#define _GUARD_CALLABLE_TYPE_1 385 +#define _GUARD_DORV_NO_DICT 386 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 387 +#define _GUARD_GLOBALS_VERSION 388 +#define _GUARD_IS_FALSE_POP 389 +#define _GUARD_IS_NONE_POP 390 +#define _GUARD_IS_NOT_NONE_POP 391 +#define _GUARD_IS_TRUE_POP 392 +#define _GUARD_KEYS_VERSION 393 +#define _GUARD_NOS_DICT 394 +#define _GUARD_NOS_FLOAT 395 +#define _GUARD_NOS_INT 396 +#define _GUARD_NOS_LIST 397 +#define _GUARD_NOS_NOT_NULL 398 +#define _GUARD_NOS_NULL 399 +#define _GUARD_NOS_TUPLE 400 +#define _GUARD_NOS_UNICODE 401 +#define _GUARD_NOT_EXHAUSTED_LIST 402 +#define _GUARD_NOT_EXHAUSTED_RANGE 403 +#define _GUARD_NOT_EXHAUSTED_TUPLE 404 +#define _GUARD_THIRD_NULL 405 +#define _GUARD_TOS_ANY_SET 406 +#define _GUARD_TOS_DICT 407 +#define _GUARD_TOS_FLOAT 408 +#define _GUARD_TOS_INT 409 +#define _GUARD_TOS_LIST 410 +#define _GUARD_TOS_SLICE 411 +#define _GUARD_TOS_TUPLE 412 +#define _GUARD_TOS_UNICODE 413 +#define _GUARD_TYPE_VERSION 414 +#define _GUARD_TYPE_VERSION_AND_LOCK 415 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 417 -#define _INIT_CALL_PY_EXACT_ARGS 418 -#define _INIT_CALL_PY_EXACT_ARGS_0 419 -#define _INIT_CALL_PY_EXACT_ARGS_1 420 -#define _INIT_CALL_PY_EXACT_ARGS_2 421 -#define _INIT_CALL_PY_EXACT_ARGS_3 422 -#define _INIT_CALL_PY_EXACT_ARGS_4 423 -#define _INSERT_NULL 424 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 416 +#define _INIT_CALL_PY_EXACT_ARGS 417 +#define _INIT_CALL_PY_EXACT_ARGS_0 418 +#define _INIT_CALL_PY_EXACT_ARGS_1 419 +#define _INIT_CALL_PY_EXACT_ARGS_2 420 +#define _INIT_CALL_PY_EXACT_ARGS_3 421 +#define _INIT_CALL_PY_EXACT_ARGS_4 422 +#define _INSERT_NULL 423 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -179,171 +178,170 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 425 +#define _IS_NONE 424 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 426 -#define _ITER_CHECK_RANGE 427 -#define _ITER_CHECK_TUPLE 428 -#define _ITER_JUMP_LIST 429 -#define _ITER_JUMP_RANGE 430 -#define _ITER_JUMP_TUPLE 431 -#define _ITER_NEXT_LIST 432 -#define _ITER_NEXT_LIST_TIER_TWO 433 -#define _ITER_NEXT_RANGE 434 -#define _ITER_NEXT_TUPLE 435 -#define _JUMP_TO_TOP 436 +#define _ITER_CHECK_LIST 425 +#define _ITER_CHECK_RANGE 426 +#define _ITER_CHECK_TUPLE 427 +#define _ITER_JUMP_LIST 428 +#define _ITER_JUMP_RANGE 429 +#define _ITER_JUMP_TUPLE 430 +#define _ITER_NEXT_LIST 431 +#define _ITER_NEXT_LIST_TIER_TWO 432 +#define _ITER_NEXT_RANGE 433 +#define _ITER_NEXT_TUPLE 434 +#define _JUMP_TO_TOP 435 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 437 -#define _LOAD_ATTR_CLASS 438 +#define _LOAD_ATTR 436 +#define _LOAD_ATTR_CLASS 437 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 439 -#define _LOAD_ATTR_METHOD_LAZY_DICT 440 -#define _LOAD_ATTR_METHOD_NO_DICT 441 -#define _LOAD_ATTR_METHOD_WITH_VALUES 442 -#define _LOAD_ATTR_MODULE 443 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 444 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 445 -#define _LOAD_ATTR_PROPERTY_FRAME 446 -#define _LOAD_ATTR_SLOT 447 -#define _LOAD_ATTR_WITH_HINT 448 +#define _LOAD_ATTR_INSTANCE_VALUE 438 +#define _LOAD_ATTR_METHOD_LAZY_DICT 439 +#define _LOAD_ATTR_METHOD_NO_DICT 440 +#define _LOAD_ATTR_METHOD_WITH_VALUES 441 +#define _LOAD_ATTR_MODULE 442 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 443 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 444 +#define _LOAD_ATTR_PROPERTY_FRAME 445 +#define _LOAD_ATTR_SLOT 446 +#define _LOAD_ATTR_WITH_HINT 447 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 449 +#define _LOAD_BYTECODE 448 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 450 -#define _LOAD_CONST_INLINE_BORROW 451 -#define _LOAD_CONST_UNDER_INLINE 452 -#define _LOAD_CONST_UNDER_INLINE_BORROW 453 +#define _LOAD_CONST_INLINE 449 +#define _LOAD_CONST_INLINE_BORROW 450 +#define _LOAD_CONST_UNDER_INLINE 451 +#define _LOAD_CONST_UNDER_INLINE_BORROW 452 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 454 -#define _LOAD_FAST_0 455 -#define _LOAD_FAST_1 456 -#define _LOAD_FAST_2 457 -#define _LOAD_FAST_3 458 -#define _LOAD_FAST_4 459 -#define _LOAD_FAST_5 460 -#define _LOAD_FAST_6 461 -#define _LOAD_FAST_7 462 +#define _LOAD_FAST 453 +#define _LOAD_FAST_0 454 +#define _LOAD_FAST_1 455 +#define _LOAD_FAST_2 456 +#define _LOAD_FAST_3 457 +#define _LOAD_FAST_4 458 +#define _LOAD_FAST_5 459 +#define _LOAD_FAST_6 460 +#define _LOAD_FAST_7 461 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 463 -#define _LOAD_FAST_BORROW_0 464 -#define _LOAD_FAST_BORROW_1 465 -#define _LOAD_FAST_BORROW_2 466 -#define _LOAD_FAST_BORROW_3 467 -#define _LOAD_FAST_BORROW_4 468 -#define _LOAD_FAST_BORROW_5 469 -#define _LOAD_FAST_BORROW_6 470 -#define _LOAD_FAST_BORROW_7 471 +#define _LOAD_FAST_BORROW 462 +#define _LOAD_FAST_BORROW_0 463 +#define _LOAD_FAST_BORROW_1 464 +#define _LOAD_FAST_BORROW_2 465 +#define _LOAD_FAST_BORROW_3 466 +#define _LOAD_FAST_BORROW_4 467 +#define _LOAD_FAST_BORROW_5 468 +#define _LOAD_FAST_BORROW_6 469 +#define _LOAD_FAST_BORROW_7 470 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 472 -#define _LOAD_GLOBAL_BUILTINS 473 -#define _LOAD_GLOBAL_MODULE 474 +#define _LOAD_GLOBAL 471 +#define _LOAD_GLOBAL_BUILTINS 472 +#define _LOAD_GLOBAL_MODULE 473 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 475 -#define _LOAD_SMALL_INT_0 476 -#define _LOAD_SMALL_INT_1 477 -#define _LOAD_SMALL_INT_2 478 -#define _LOAD_SMALL_INT_3 479 -#define _LOAD_SPECIAL 480 +#define _LOAD_SMALL_INT 474 +#define _LOAD_SMALL_INT_0 475 +#define _LOAD_SMALL_INT_1 476 +#define _LOAD_SMALL_INT_2 477 +#define _LOAD_SMALL_INT_3 478 +#define _LOAD_SPECIAL 479 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 481 +#define _MAKE_CALLARGS_A_TUPLE 480 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 482 +#define _MAKE_WARM 481 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 483 -#define _MAYBE_EXPAND_METHOD_KW 484 -#define _MONITOR_CALL 485 -#define _MONITOR_CALL_KW 486 -#define _MONITOR_JUMP_BACKWARD 487 -#define _MONITOR_RESUME 488 +#define _MAYBE_EXPAND_METHOD 482 +#define _MAYBE_EXPAND_METHOD_KW 483 +#define _MONITOR_CALL 484 +#define _MONITOR_CALL_KW 485 +#define _MONITOR_JUMP_BACKWARD 486 +#define _MONITOR_RESUME 487 #define _NOP NOP -#define _POP_CALL 489 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 490 -#define _POP_CALL_ONE 491 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 492 -#define _POP_CALL_TWO 493 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 494 +#define _POP_CALL 488 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 489 +#define _POP_CALL_ONE 490 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 491 +#define _POP_CALL_TWO 492 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 493 #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 495 -#define _POP_JUMP_IF_TRUE 496 +#define _POP_JUMP_IF_FALSE 494 +#define _POP_JUMP_IF_TRUE 495 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 497 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 498 -#define _POP_TWO 499 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 500 +#define _POP_TOP_LOAD_CONST_INLINE 496 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 497 +#define _POP_TWO 498 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 499 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 501 +#define _PUSH_FRAME 500 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 502 -#define _PY_FRAME_GENERAL 503 -#define _PY_FRAME_KW 504 -#define _QUICKEN_RESUME 505 -#define _REPLACE_WITH_TRUE 506 +#define _PUSH_NULL_CONDITIONAL 501 +#define _PY_FRAME_GENERAL 502 +#define _PY_FRAME_KW 503 +#define _QUICKEN_RESUME 504 +#define _REPLACE_WITH_TRUE 505 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 507 -#define _SEND 508 -#define _SEND_GEN_FRAME 509 +#define _SAVE_RETURN_OFFSET 506 +#define _SEND 507 +#define _SEND_GEN_FRAME 508 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 510 -#define _STORE_ATTR 511 -#define _STORE_ATTR_INSTANCE_VALUE 512 -#define _STORE_ATTR_SLOT 513 -#define _STORE_ATTR_WITH_HINT 514 +#define _START_EXECUTOR 509 +#define _STORE_ATTR 510 +#define _STORE_ATTR_INSTANCE_VALUE 511 +#define _STORE_ATTR_SLOT 512 +#define _STORE_ATTR_WITH_HINT 513 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 515 -#define _STORE_FAST_0 516 -#define _STORE_FAST_1 517 -#define _STORE_FAST_2 518 -#define _STORE_FAST_3 519 -#define _STORE_FAST_4 520 -#define _STORE_FAST_5 521 -#define _STORE_FAST_6 522 -#define _STORE_FAST_7 523 +#define _STORE_FAST 514 +#define _STORE_FAST_0 515 +#define _STORE_FAST_1 516 +#define _STORE_FAST_2 517 +#define _STORE_FAST_3 518 +#define _STORE_FAST_4 519 +#define _STORE_FAST_5 520 +#define _STORE_FAST_6 521 +#define _STORE_FAST_7 522 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 524 -#define _STORE_SUBSCR 525 -#define _STORE_SUBSCR_DICT 526 -#define _STORE_SUBSCR_LIST_INT 527 -#define _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS 528 +#define _STORE_SLICE 523 +#define _STORE_SUBSCR 524 +#define _STORE_SUBSCR_DICT 525 +#define _STORE_SUBSCR_LIST_INT 526 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 529 -#define _TO_BOOL 530 +#define _TIER2_RESUME_CHECK 527 +#define _TO_BOOL 528 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 531 +#define _TO_BOOL_LIST 529 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 532 +#define _TO_BOOL_STR 530 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 533 -#define _UNPACK_SEQUENCE_LIST 534 -#define _UNPACK_SEQUENCE_TUPLE 535 -#define _UNPACK_SEQUENCE_TWO_TUPLE 536 +#define _UNPACK_SEQUENCE 531 +#define _UNPACK_SEQUENCE_LIST 532 +#define _UNPACK_SEQUENCE_TUPLE 533 +#define _UNPACK_SEQUENCE_TWO_TUPLE 534 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 536 +#define MAX_UOP_ID 534 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index dcbbdb7fc6a8fc..6b298211949cc9 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -102,7 +102,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, @@ -117,7 +116,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -350,7 +348,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", - [_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", @@ -623,7 +620,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR] = "_STORE_SUBSCR", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", - [_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS] = "_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS", [_SWAP] = "_SWAP", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", @@ -812,8 +808,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 4; case _BINARY_OP_SUBSCR_LIST_INT: return 2; - case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: - return 2; case _BINARY_OP_SUBSCR_LIST_SLICE: return 2; case _BINARY_OP_SUBSCR_STR_INT: @@ -842,8 +836,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_LIST_INT: return 3; - case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: - return 3; case _STORE_SUBSCR_DICT: return 3; case _DELETE_SUBSCR: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 7ccbc817bd280f..8c6b5639e86624 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1707,10 +1707,10 @@ def f(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertEqual(uops.count("_GUARD_NOS_LIST"), 0) - self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS"), 1) + self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT"), 1) self.assertEqual(uops.count("_GUARD_TOS_LIST"), 0) self.assertEqual(uops.count("_UNPACK_SEQUENCE_LIST"), 1) - self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT"), 1) self.assertEqual(uops.count("_TO_BOOL_LIST"), 1) def test_remove_guard_for_known_type_set(self): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 77211b63ce0739..58453549381dd9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -907,32 +907,6 @@ dummy_func( DECREF_INPUTS(); } - op(_BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS, (list_st, sub_st -- res)) { - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; -#ifdef Py_GIL_DISABLED - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - DEOPT_IF(res_o == NULL); - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); -#else - DEOPT_IF(index >= PyList_GET_SIZE(list)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); -#endif - STAT_INC(BINARY_OP, hit); - INPUTS_DEAD(); - } - macro(BINARY_OP_SUBSCR_LIST_SLICE) = _GUARD_TOS_SLICE + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_SLICE; @@ -1135,34 +1109,6 @@ dummy_func( Py_DECREF(old_value); } - op(_STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS, (value, list_st, sub_st -- )) { - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list)); - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - DEOPT_IF(true); - } - STAT_INC(STORE_SUBSCR, hit); - - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); // unlock before decrefs! - DEAD(sub_st); - DEAD(list_st); - Py_DECREF(old_value); - } - macro(STORE_SUBSCR_DICT) = _GUARD_NOS_DICT + unused/1 + _STORE_SUBSCR_DICT; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index aae427e4b64b96..c9fed6974b560f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1384,48 +1384,6 @@ break; } - case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef res; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index >= PyList_GET_SIZE(list)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(BINARY_OP, hit); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - case _BINARY_OP_SUBSCR_LIST_SLICE: { _PyStackRef sub_st; _PyStackRef list_st; @@ -1785,47 +1743,6 @@ break; } - case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef value; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - break; - } - case _STORE_SUBSCR_DICT: { _PyStackRef sub; _PyStackRef dict_st; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6270cd5be71a94..72e522da1f80d0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -448,8 +448,6 @@ const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, - [_STORE_SUBSCR_LIST_INT] = _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS, - [_BINARY_OP_SUBSCR_LIST_INT] = _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS, }; /* 1 for success, 0 for not ready, cannot error at the moment. */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 3f72d73abcfb52..aa90d6b7645259 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -104,15 +104,6 @@ dummy_func(void) { GETLOCAL(oparg) = value; } - op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) { - // Can't move this to the optimizer generator for now, as it requires list_st and sub_st - // to be borrowed, but not value. - // Alternatively, we could just stricten it and require all to be borrowed. - if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { - REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); - } - } - op(_PUSH_NULL, (-- res)) { res = sym_new_null(ctx); } @@ -403,14 +394,6 @@ dummy_func(void) { ctx->done = true; } - op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) { - // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { - REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); - } - res = sym_new_not_null(ctx); - } - op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { res = sym_new_type(ctx, &PyUnicode_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8419eefbf7b6e0..3ff4e056e7dac8 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -617,22 +617,6 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *list_st; - JitOptSymbol *res; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { - REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); - } - res = sym_new_not_null(ctx); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _BINARY_OP_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { JitOptSymbol *res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; @@ -775,19 +759,6 @@ } case _STORE_SUBSCR_LIST_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *list_st; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - if (sym_is_skip_refcount(ctx, list_st) && sym_is_skip_refcount(ctx, sub_st)) { - REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _STORE_SUBSCR_LIST_INT__NO_DECREF_INPUTS: { stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; From f4567404f5c5268d4756bb081f45ffc343f06516 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 23 May 2025 14:54:13 +0000 Subject: [PATCH 05/24] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst new file mode 100644 index 00000000000000..5f9e1553ae7ca5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst @@ -0,0 +1 @@ +Add a reference count elimination pass to the JIT compiler. Patch by Ken Jin. From 5c429b60831de05ecb3ba21e9ec3dcee50de724f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 02:50:28 +0800 Subject: [PATCH 06/24] Rename things to make things clearer --- Include/internal/pycore_optimizer.h | 18 ++++++++--------- Python/optimizer_analysis.c | 6 +++--- Python/optimizer_bytecodes.c | 14 ++++++------- Python/optimizer_cases.c.h | 14 ++++++------- Python/optimizer_symbols.c | 31 +++++++++++++++-------------- 5 files changed, 41 insertions(+), 42 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e8b61096f477a1..99b288d5e4efd2 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -180,8 +180,8 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; -#define DONT_SKIP_REFCOUNT 0 -#define SKIP_REFCOUNT 1 +#define WE_MIGHT_BE_THE_ONLY_STRONG_REF 0 +#define SOMEONE_ELSE_HAS_A_VALID_STRONG_REF 1 typedef struct _jit_opt_known_class { struct { @@ -231,11 +231,9 @@ typedef struct { typedef union _jit_opt_symbol { struct { uint8_t tag; - // Whether this object skips refcount on the stack - // (using the _PyStackRef API), or not. - // 0 - normal refcounting - // 1 - skip refcounting - int8_t skip_refcount; + // Whether this object has a strong reference + // being held to it by someone else. + int8_t strong_ref_held_by_someone_else; }; JitOptKnownClass cls; JitOptKnownValue value; @@ -308,9 +306,9 @@ extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); -extern void _Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); -extern bool _Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_strong_ref_might_be_this_sym(JitOptContext *ctx, JitOptSymbol *sym); +extern bool _Py_uop_sym_is_strong_ref_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_strong_ref_is_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0ee45526a68406..0fd09cbe81f8f0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -342,9 +342,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal #define sym_new_truthiness _Py_uop_sym_new_truthiness -#define sym_set_skip_refcount _Py_uop_sym_set_skip_refcount -#define sym_set_dont_skip_refcount _Py_uop_sym_set_dont_skip_refcount -#define sym_is_skip_refcount _Py_uop_sym_is_skip_refcount +#define sym_set_strong_ref_is_held_by_someone_else _Py_uop_sym_set_strong_ref_is_held_by_someone_else +#define sym_set_strong_ref_might_be_this_sym _Py_uop_sym_set_strong_ref_might_be_this_sym +#define sym_is_strong_ref_held_by_someone_else _Py_uop_sym_is_strong_ref_held_by_someone_else static int optimize_to_bool( diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 41cde50aed1206..d2583de953b132 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -80,24 +80,24 @@ dummy_func(void) { if (sym_is_null(value)) { ctx->done = true; } - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); } op(_LOAD_FAST_BORROW, (-- value)) { value = GETLOCAL(oparg); - sym_set_skip_refcount(ctx, value); + sym_set_strong_ref_is_held_by_someone_else(ctx, value); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); } op(_STORE_FAST, (value --)) { @@ -301,7 +301,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } @@ -325,7 +325,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } @@ -349,7 +349,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index e6d191889b4476..b6908029baac8a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -31,7 +31,7 @@ if (sym_is_null(value)) { ctx->done = true; } - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -41,7 +41,7 @@ case _LOAD_FAST: { JitOptSymbol *value; value = GETLOCAL(oparg); - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -51,7 +51,7 @@ case _LOAD_FAST_BORROW: { JitOptSymbol *value; value = GETLOCAL(oparg); - sym_set_skip_refcount(ctx, value); + sym_set_strong_ref_is_held_by_someone_else(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -63,7 +63,7 @@ value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_dont_skip_refcount(ctx, value); + sym_set_strong_ref_might_be_this_sym(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -444,7 +444,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -476,7 +476,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -508,7 +508,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 68f1ecd1d316a4..3cbad86eb61df2 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -99,7 +99,7 @@ sym_new(JitOptContext *ctx) } ctx->t_arena.ty_curr_number++; self->tag = JIT_SYM_UNKNOWN_TAG; - self->skip_refcount = DONT_SKIP_REFCOUNT; + self->strong_ref_held_by_someone_else = WE_MIGHT_BE_THE_ONLY_STRONG_REF; return self; } @@ -109,7 +109,7 @@ static void make_const(JitOptSymbol *sym, PyObject *val) sym->value.value = Py_NewRef(val); // Constants don't need to be refcounted, as they are always // kept alive by co_consts. - sym->skip_refcount = SKIP_REFCOUNT; + sym->strong_ref_held_by_someone_else = SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; } static inline void @@ -386,22 +386,23 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_strong_ref_is_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym) { - sym->skip_refcount = SKIP_REFCOUNT; + sym->strong_ref_held_by_someone_else = SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; } void -_Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_strong_ref_might_be_this_sym(JitOptContext *ctx, JitOptSymbol *sym) { - sym->skip_refcount = DONT_SKIP_REFCOUNT; + sym->strong_ref_held_by_someone_else = WE_MIGHT_BE_THE_ONLY_STRONG_REF; } bool -_Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_is_strong_ref_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym) { - assert(sym->skip_refcount == SKIP_REFCOUNT || sym->skip_refcount == DONT_SKIP_REFCOUNT); - return sym->skip_refcount == SKIP_REFCOUNT; + assert(sym->strong_ref_held_by_someone_else == SOMEONE_ELSE_HAS_A_VALID_STRONG_REF || + sym->strong_ref_held_by_someone_else == WE_MIGHT_BE_THE_ONLY_STRONG_REF); + return sym->strong_ref_held_by_someone_else == SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; } @@ -794,7 +795,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "top is refcounted"); + TEST_PREDICATE(!_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "top is refcounted"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); @@ -843,7 +844,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "42 isn't refcounted"); + TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "42 isn't refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); @@ -876,12 +877,12 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "type should be refcounted"); - _Py_uop_sym_set_skip_refcount(ctx, sym); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "type should not be refcounted"); + TEST_PREDICATE(!_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "type should be refcounted"); + _Py_uop_sym_set_strong_ref_is_held_by_someone_else(ctx, sym); + TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "type should not be refcounted"); sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "None should not be refcounted"); + TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "None should not be refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); sym = _Py_uop_sym_new_const(ctx, Py_False); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); From 15351331857d66fbdc965c8faa4f46ecb32aefa8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 02:52:27 +0800 Subject: [PATCH 07/24] Revert "Rename things to make things clearer" This reverts commit 5c429b60831de05ecb3ba21e9ec3dcee50de724f. --- Include/internal/pycore_optimizer.h | 18 +++++++++-------- Python/optimizer_analysis.c | 6 +++--- Python/optimizer_bytecodes.c | 14 ++++++------- Python/optimizer_cases.c.h | 14 ++++++------- Python/optimizer_symbols.c | 31 ++++++++++++++--------------- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 99b288d5e4efd2..e8b61096f477a1 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -180,8 +180,8 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; -#define WE_MIGHT_BE_THE_ONLY_STRONG_REF 0 -#define SOMEONE_ELSE_HAS_A_VALID_STRONG_REF 1 +#define DONT_SKIP_REFCOUNT 0 +#define SKIP_REFCOUNT 1 typedef struct _jit_opt_known_class { struct { @@ -231,9 +231,11 @@ typedef struct { typedef union _jit_opt_symbol { struct { uint8_t tag; - // Whether this object has a strong reference - // being held to it by someone else. - int8_t strong_ref_held_by_someone_else; + // Whether this object skips refcount on the stack + // (using the _PyStackRef API), or not. + // 0 - normal refcounting + // 1 - skip refcounting + int8_t skip_refcount; }; JitOptKnownClass cls; JitOptKnownValue value; @@ -306,9 +308,9 @@ extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); -extern void _Py_uop_sym_set_strong_ref_might_be_this_sym(JitOptContext *ctx, JitOptSymbol *sym); -extern bool _Py_uop_sym_is_strong_ref_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_strong_ref_is_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern bool _Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0fd09cbe81f8f0..0ee45526a68406 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -342,9 +342,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal #define sym_new_truthiness _Py_uop_sym_new_truthiness -#define sym_set_strong_ref_is_held_by_someone_else _Py_uop_sym_set_strong_ref_is_held_by_someone_else -#define sym_set_strong_ref_might_be_this_sym _Py_uop_sym_set_strong_ref_might_be_this_sym -#define sym_is_strong_ref_held_by_someone_else _Py_uop_sym_is_strong_ref_held_by_someone_else +#define sym_set_skip_refcount _Py_uop_sym_set_skip_refcount +#define sym_set_dont_skip_refcount _Py_uop_sym_set_dont_skip_refcount +#define sym_is_skip_refcount _Py_uop_sym_is_skip_refcount static int optimize_to_bool( diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d2583de953b132..41cde50aed1206 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -80,24 +80,24 @@ dummy_func(void) { if (sym_is_null(value)) { ctx->done = true; } - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST_BORROW, (-- value)) { value = GETLOCAL(oparg); - sym_set_strong_ref_is_held_by_someone_else(ctx, value); + sym_set_skip_refcount(ctx, value); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); } op(_STORE_FAST, (value --)) { @@ -301,7 +301,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } @@ -325,7 +325,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } @@ -349,7 +349,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b6908029baac8a..e6d191889b4476 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -31,7 +31,7 @@ if (sym_is_null(value)) { ctx->done = true; } - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -41,7 +41,7 @@ case _LOAD_FAST: { JitOptSymbol *value; value = GETLOCAL(oparg); - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -51,7 +51,7 @@ case _LOAD_FAST_BORROW: { JitOptSymbol *value; value = GETLOCAL(oparg); - sym_set_strong_ref_is_held_by_someone_else(ctx, value); + sym_set_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -63,7 +63,7 @@ value = GETLOCAL(oparg); JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_strong_ref_might_be_this_sym(ctx, value); + sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -444,7 +444,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -476,7 +476,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -508,7 +508,7 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_strong_ref_held_by_someone_else(ctx, left) && sym_is_strong_ref_held_by_someone_else(ctx, right)) { + if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 3cbad86eb61df2..68f1ecd1d316a4 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -99,7 +99,7 @@ sym_new(JitOptContext *ctx) } ctx->t_arena.ty_curr_number++; self->tag = JIT_SYM_UNKNOWN_TAG; - self->strong_ref_held_by_someone_else = WE_MIGHT_BE_THE_ONLY_STRONG_REF; + self->skip_refcount = DONT_SKIP_REFCOUNT; return self; } @@ -109,7 +109,7 @@ static void make_const(JitOptSymbol *sym, PyObject *val) sym->value.value = Py_NewRef(val); // Constants don't need to be refcounted, as they are always // kept alive by co_consts. - sym->strong_ref_held_by_someone_else = SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; + sym->skip_refcount = SKIP_REFCOUNT; } static inline void @@ -386,23 +386,22 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_strong_ref_is_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) { - sym->strong_ref_held_by_someone_else = SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; + sym->skip_refcount = SKIP_REFCOUNT; } void -_Py_uop_sym_set_strong_ref_might_be_this_sym(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) { - sym->strong_ref_held_by_someone_else = WE_MIGHT_BE_THE_ONLY_STRONG_REF; + sym->skip_refcount = DONT_SKIP_REFCOUNT; } bool -_Py_uop_sym_is_strong_ref_held_by_someone_else(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) { - assert(sym->strong_ref_held_by_someone_else == SOMEONE_ELSE_HAS_A_VALID_STRONG_REF || - sym->strong_ref_held_by_someone_else == WE_MIGHT_BE_THE_ONLY_STRONG_REF); - return sym->strong_ref_held_by_someone_else == SOMEONE_ELSE_HAS_A_VALID_STRONG_REF; + assert(sym->skip_refcount == SKIP_REFCOUNT || sym->skip_refcount == DONT_SKIP_REFCOUNT); + return sym->skip_refcount == SKIP_REFCOUNT; } @@ -795,7 +794,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "top is refcounted"); + TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "top is refcounted"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); @@ -844,7 +843,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "42 isn't refcounted"); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "42 isn't refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); @@ -877,12 +876,12 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "type should be refcounted"); - _Py_uop_sym_set_strong_ref_is_held_by_someone_else(ctx, sym); - TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "type should not be refcounted"); + TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "type should be refcounted"); + _Py_uop_sym_set_skip_refcount(ctx, sym); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "type should not be refcounted"); sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_is_strong_ref_held_by_someone_else(ctx, sym), "None should not be refcounted"); + TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "None should not be refcounted"); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); sym = _Py_uop_sym_new_const(ctx, Py_False); TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); From 8f6206795863595a440c2647db77f09aec72fab2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 23:26:12 +0800 Subject: [PATCH 08/24] Massive refactor from JitOptSymbol to JitRef --- Include/internal/pycore_optimizer.h | 130 +- Python/optimizer_analysis.c | 71 +- Python/optimizer_bytecodes.c | 566 +++++---- Python/optimizer_cases.c.h | 1172 +++++++++--------- Python/optimizer_symbols.c | 402 +++--- Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/optimizer_generator.py | 14 +- 7 files changed, 1189 insertions(+), 1168 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e8b61096f477a1..78cea6198d9767 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_typedefs.h" // _PyInterpreterFrame #include "pycore_uop_ids.h" +#include "pycore_stackref.h" #include @@ -180,13 +181,10 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; -#define DONT_SKIP_REFCOUNT 0 -#define SKIP_REFCOUNT 1 typedef struct _jit_opt_known_class { struct { uint8_t tag; - uint8_t skip_refcount; }; uint32_t version; PyTypeObject *type; @@ -195,7 +193,6 @@ typedef struct _jit_opt_known_class { typedef struct _jit_opt_known_version { struct { uint8_t tag; - uint8_t skip_refcount; }; uint32_t version; } JitOptKnownVersion; @@ -203,7 +200,6 @@ typedef struct _jit_opt_known_version { typedef struct _jit_opt_known_value { struct { uint8_t tag; - uint8_t skip_refcount; }; PyObject *value; } JitOptKnownValue; @@ -213,7 +209,6 @@ typedef struct _jit_opt_known_value { typedef struct _jit_opt_tuple { struct { uint8_t tag; - uint8_t skip_refcount; }; uint8_t length; uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; @@ -222,7 +217,6 @@ typedef struct _jit_opt_tuple { typedef struct { struct { uint8_t tag; - uint8_t skip_refcount; }; bool invert; uint16_t value; @@ -231,11 +225,6 @@ typedef struct { typedef union _jit_opt_symbol { struct { uint8_t tag; - // Whether this object skips refcount on the stack - // (using the _PyStackRef API), or not. - // 0 - normal refcounting - // 1 - skip refcounting - int8_t skip_refcount; }; JitOptKnownClass cls; JitOptKnownValue value; @@ -245,15 +234,65 @@ typedef union _jit_opt_symbol { } JitOptSymbol; +// This mimics the _PyStackRef API +typedef union { + uintptr_t bits; +} JitOptRef; + +#define JIT_BITS_TO_PTR(REF) ((JitOptSymbol *)((REF).bits)) +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~Py_TAG_REFCNT))) + +static inline JitOptSymbol * +PyJitRef_AsSymbolBorrow(JitOptRef ref) +{ + return JIT_BITS_TO_PTR_MASKED(ref); +} + +bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); + +static inline JitOptRef +PyJitRef_FromSymbolSteal(JitOptSymbol *sym) +{ + if (sym == NULL || _Py_uop_sym_is_immortal(sym)) { + return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; + } + return (JitOptRef){.bits=(uintptr_t)sym}; +} + +static inline JitOptRef +PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) +{ + return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; +} + +static inline JitOptRef +PyJitRef_Borrow(JitOptRef ref) +{ + return (JitOptRef){ .bits = ref.bits | Py_TAG_REFCNT }; +} + +static const JitOptRef PyJitRef_NULL = { .bits = PyStackRef_NULL_BITS }; + +static inline bool +PyJitRef_IsNull(JitOptRef ref) +{ + return ref.bits == PyStackRef_NULL_BITS; +} + +static inline int +PyJitRef_IsBorrowed(JitOptRef ref) +{ + return (ref.bits & Py_TAG_REFCNT) == Py_TAG_REFCNT; +} struct _Py_UOpsAbstractFrame { // Max stacklen int stack_len; int locals_len; - JitOptSymbol **stack_pointer; - JitOptSymbol **stack; - JitOptSymbol **locals; + JitOptRef *stack_pointer; + JitOptRef *stack; + JitOptRef *locals; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -276,41 +315,36 @@ typedef struct _JitOptContext { // Arena for the symbolic types. ty_arena t_arena; - JitOptSymbol **n_consumed; - JitOptSymbol **limit; - JitOptSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + JitOptRef *n_consumed; + JitOptRef *limit; + JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; } JitOptContext; -extern bool _Py_uop_sym_is_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym); -extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx); -extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx); -extern JitOptSymbol *_Py_uop_sym_new_type( +extern bool _Py_uop_ref_is_null(JitOptRef sym); +extern bool _Py_uop_ref_is_not_null(JitOptRef sym); +extern bool _Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef sym); +extern PyObject *_Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef sym); +extern JitOptRef _Py_uop_ref_new_unknown(JitOptContext *ctx); +extern JitOptRef _Py_uop_ref_new_not_null(JitOptContext *ctx); +extern JitOptRef _Py_uop_ref_new_type( JitOptContext *ctx, PyTypeObject *typ); -extern JitOptSymbol *_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); -extern JitOptSymbol *_Py_uop_sym_new_null(JitOptContext *ctx); -extern bool _Py_uop_sym_has_type(JitOptSymbol *sym); -extern bool _Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val); -extern bool _Py_uop_sym_is_bottom(JitOptSymbol *sym); -extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym); -extern PyTypeObject *_Py_uop_sym_get_type(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args); -extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item); -extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); - -extern void _Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); -extern bool _Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym); +extern JitOptRef _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val); +extern JitOptRef _Py_uop_ref_new_null(JitOptContext *ctx); +extern bool _Py_uop_ref_has_type(JitOptRef sym); +extern bool _Py_uop_ref_matches_type(JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_ref_matches_type_version(JitOptRef sym, unsigned int version); +extern void _Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef sym, unsigned int version); +extern void _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef sym, PyObject *const_val); +extern bool _Py_uop_ref_is_bottom(JitOptRef sym); +extern int _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef sym); +extern PyTypeObject *_Py_uop_ref_get_type(JitOptRef sym); +extern JitOptRef _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); +extern JitOptRef _Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item); +extern int _Py_uop_ref_tuple_length(JitOptRef sym); +extern JitOptRef _Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); @@ -319,7 +353,7 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len); extern int _Py_uop_frame_pop(JitOptContext *ctx); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0ee45526a68406..874d4482f24f5f 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -315,54 +315,51 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, INST->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ -#define sym_is_not_null _Py_uop_sym_is_not_null -#define sym_is_const _Py_uop_sym_is_const -#define sym_get_const _Py_uop_sym_get_const -#define sym_new_unknown _Py_uop_sym_new_unknown -#define sym_new_not_null _Py_uop_sym_new_not_null -#define sym_new_type _Py_uop_sym_new_type -#define sym_is_null _Py_uop_sym_is_null -#define sym_new_const _Py_uop_sym_new_const -#define sym_new_null _Py_uop_sym_new_null -#define sym_has_type _Py_uop_sym_has_type -#define sym_get_type _Py_uop_sym_get_type -#define sym_matches_type _Py_uop_sym_matches_type -#define sym_matches_type_version _Py_uop_sym_matches_type_version -#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) -#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) -#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) -#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) -#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) -#define sym_is_bottom _Py_uop_sym_is_bottom -#define sym_truthiness _Py_uop_sym_truthiness +#define ref_is_not_null _Py_uop_ref_is_not_null +#define ref_is_const _Py_uop_ref_is_const +#define ref_get_const _Py_uop_ref_get_const +#define ref_new_unknown _Py_uop_ref_new_unknown +#define ref_new_not_null _Py_uop_ref_new_not_null +#define ref_new_type _Py_uop_ref_new_type +#define ref_is_null _Py_uop_ref_is_null +#define ref_new_const _Py_uop_ref_new_const +#define ref_new_null _Py_uop_ref_new_null +#define ref_has_type _Py_uop_ref_has_type +#define ref_get_type _Py_uop_ref_get_type +#define ref_matches_type _Py_uop_ref_matches_type +#define ref_matches_type_version _Py_uop_ref_matches_type_version +#define ref_set_null(SYM) _Py_uop_ref_set_null(ctx, SYM) +#define ref_set_non_null(SYM) _Py_uop_ref_set_non_null(ctx, SYM) +#define ref_set_type(SYM, TYPE) _Py_uop_ref_set_type(ctx, SYM, TYPE) +#define ref_set_type_version(SYM, VERSION) _Py_uop_ref_set_type_version(ctx, SYM, VERSION) +#define ref_set_const(SYM, CNST) _Py_uop_ref_set_const(ctx, SYM, CNST) +#define ref_is_bottom _Py_uop_ref_is_bottom +#define ref_truthiness _Py_uop_ref_truthiness #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop -#define sym_new_tuple _Py_uop_sym_new_tuple -#define sym_tuple_getitem _Py_uop_sym_tuple_getitem -#define sym_tuple_length _Py_uop_sym_tuple_length -#define sym_is_immortal _Py_uop_sym_is_immortal -#define sym_new_truthiness _Py_uop_sym_new_truthiness -#define sym_set_skip_refcount _Py_uop_sym_set_skip_refcount -#define sym_set_dont_skip_refcount _Py_uop_sym_set_dont_skip_refcount -#define sym_is_skip_refcount _Py_uop_sym_is_skip_refcount +#define ref_new_tuple _Py_uop_ref_new_tuple +#define ref_tuple_getitem _Py_uop_ref_tuple_getitem +#define ref_tuple_length _Py_uop_ref_tuple_length +#define ref_is_immortal _Py_uop_ref_is_immortal +#define ref_new_truthiness _Py_uop_ref_new_truthiness static int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, - JitOptSymbol *value, - JitOptSymbol **result_ptr) + JitOptRef value, + JitOptRef *result_ptr) { - if (sym_matches_type(value, &PyBool_Type)) { + if (ref_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); *result_ptr = value; return 1; } - int truthiness = sym_truthiness(ctx, value); + int truthiness = ref_truthiness(ctx, value); if (truthiness >= 0) { PyObject *load = truthiness ? Py_True : Py_False; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - *result_ptr = sym_new_const(ctx, load); + *result_ptr = ref_new_const(ctx, load); return 1; } return 0; @@ -378,7 +375,7 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) } } -static JitOptSymbol * +static JitOptRef lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, PyTypeObject *type, PyObject *name, uint16_t immortal, uint16_t mortal) @@ -389,10 +386,10 @@ lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup); - return sym_new_const(ctx, lookup); + return ref_new_const(ctx, lookup); } } - return sym_new_not_null(ctx); + return ref_new_not_null(ctx); } /* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a @@ -487,7 +484,7 @@ optimize_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptSymbol **stack_pointer = ctx->frame->stack_pointer; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG if (get_lltrace() >= 3) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 41cde50aed1206..333971a2e02828 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -9,32 +9,32 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; /* Shortened forms for convenience */ -#define sym_is_not_null _Py_uop_sym_is_not_null -#define sym_is_const _Py_uop_sym_is_const -#define sym_get_const _Py_uop_sym_get_const -#define sym_new_unknown _Py_uop_sym_new_unknown -#define sym_new_not_null _Py_uop_sym_new_not_null -#define sym_new_type _Py_uop_sym_new_type -#define sym_is_null _Py_uop_sym_is_null -#define sym_new_const _Py_uop_sym_new_const -#define sym_new_null _Py_uop_sym_new_null -#define sym_matches_type _Py_uop_sym_matches_type -#define sym_matches_type_version _Py_uop_sym_matches_type_version -#define sym_get_type _Py_uop_sym_get_type -#define sym_has_type _Py_uop_sym_has_type -#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) -#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) -#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) -#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) -#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) -#define sym_is_bottom _Py_uop_sym_is_bottom +#define ref_is_not_null _Py_uop_ref_is_not_null +#define ref_is_const _Py_uop_ref_is_const +#define ref_get_const _Py_uop_ref_get_const +#define ref_new_unknown _Py_uop_ref_new_unknown +#define ref_new_not_null _Py_uop_ref_new_not_null +#define ref_new_type _Py_uop_ref_new_type +#define ref_is_null _Py_uop_ref_is_null +#define ref_new_const _Py_uop_ref_new_const +#define ref_new_null _Py_uop_ref_new_null +#define ref_matches_type _Py_uop_ref_matches_type +#define ref_matches_type_version _Py_uop_ref_matches_type_version +#define ref_get_type _Py_uop_ref_get_type +#define ref_has_type _Py_uop_ref_has_type +#define ref_set_null(SYM) _Py_uop_ref_set_null(ctx, SYM) +#define ref_set_non_null(SYM) _Py_uop_ref_set_non_null(ctx, SYM) +#define ref_set_type(SYM, TYPE) _Py_uop_ref_set_type(ctx, SYM, TYPE) +#define ref_set_type_version(SYM, VERSION) _Py_uop_ref_set_type_version(ctx, SYM, VERSION) +#define ref_set_const(SYM, CNST) _Py_uop_ref_set_const(ctx, SYM, CNST) +#define ref_is_bottom _Py_uop_ref_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop -#define sym_new_tuple _Py_uop_sym_new_tuple -#define sym_tuple_getitem _Py_uop_sym_tuple_getitem -#define sym_tuple_length _Py_uop_sym_tuple_length -#define sym_is_immortal _Py_uop_sym_is_immortal -#define sym_new_truthiness _Py_uop_sym_new_truthiness +#define ref_new_tuple _Py_uop_ref_new_tuple +#define ref_tuple_getitem _Py_uop_ref_tuple_getitem +#define ref_tuple_length _Py_uop_ref_tuple_length +#define ref_is_immortal _Py_uop_ref_is_immortal +#define ref_new_truthiness _Py_uop_ref_new_truthiness extern int optimize_to_bool( @@ -77,27 +77,23 @@ dummy_func(void) { op(_LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { + if (ref_is_null(value)) { ctx->done = true; } - sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); - sym_set_dont_skip_refcount(ctx, value); } op(_LOAD_FAST_BORROW, (-- value)) { - value = GETLOCAL(oparg); - sym_set_skip_refcount(ctx, value); + value = PyJitRef_Borrow(GETLOCAL(oparg)); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = ref_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_dont_skip_refcount(ctx, value); } op(_STORE_FAST, (value --)) { @@ -105,38 +101,38 @@ dummy_func(void) { } op(_PUSH_NULL, (-- res)) { - res = sym_new_null(ctx); + res = ref_new_null(ctx); } op(_GUARD_TOS_INT, (value -- value)) { - if (sym_matches_type(value, &PyLong_Type)) { + if (ref_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyLong_Type); + ref_set_type(value, &PyLong_Type); } op(_GUARD_NOS_INT, (left, unused -- left, unused)) { - if (sym_matches_type(left, &PyLong_Type)) { + if (ref_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); + ref_set_type(left, &PyLong_Type); } op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { - if (type == sym_get_const(ctx, owner)) { + if (type == ref_get_const(ctx, owner)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { - sym_set_const(owner, type); + ref_set_const(owner, type); } } } op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { assert(type_version); - if (sym_matches_type_version(owner, type_version)) { + if (ref_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { // add watcher so that whenever the type changes we invalidate this @@ -148,7 +144,7 @@ dummy_func(void) { // if it wasn't this means that the type version was previously set to something else // and we set the owner to bottom, so we don't need to add a watcher because we must have // already added one earlier. - if (sym_set_type_version(owner, type_version)) { + if (ref_set_type_version(owner, type_version)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); } @@ -158,27 +154,27 @@ dummy_func(void) { } op(_GUARD_TOS_FLOAT, (value -- value)) { - if (sym_matches_type(value, &PyFloat_Type)) { + if (ref_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyFloat_Type); + ref_set_type(value, &PyFloat_Type); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { - if (sym_matches_type(left, &PyFloat_Type)) { + if (ref_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyFloat_Type); + ref_set_type(left, &PyFloat_Type); } op(_BINARY_OP, (lhs, rhs -- res)) { - bool lhs_int = sym_matches_type(lhs, &PyLong_Type); - bool rhs_int = sym_matches_type(rhs, &PyLong_Type); - bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); - bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); + bool lhs_int = ref_matches_type(lhs, &PyLong_Type); + bool rhs_int = ref_matches_type(rhs, &PyLong_Type); + bool lhs_float = ref_matches_type(lhs, &PyFloat_Type); + bool rhs_float = ref_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { // There's something other than an int or float involved: - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { // This one's fun... the *type* of the result depends on the @@ -195,195 +191,195 @@ dummy_func(void) { if (rhs_float) { // Case D, E, F, or G... can't know without the sign of the LHS // or whether the RHS is whole, which isn't worth the effort: - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } else if (lhs_float) { // Case C: - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, rhs)) { + else if (!ref_is_const(ctx, rhs)) { // Case A or B... can't know without the sign of the RHS: - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { + else if (_PyLong_IsNegative((PyLongObject *)ref_get_const(ctx, rhs))) { // Case B: - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } else { // Case A: - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } else if (lhs_int && rhs_int) { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); + assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { - JitOptSymbol *res; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + JitOptRef res; + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); + assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } // _STORE_FAST: GETLOCAL(this_instr->operand0) = res; @@ -395,109 +391,109 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) { - assert(sym_matches_type(tuple_st, &PyTuple_Type)); - if (sym_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); - long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); + assert(ref_matches_type(tuple_st, &PyTuple_Type)); + if (ref_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(ref_get_const(ctx, sub_st))); + long index = PyLong_AsLong(ref_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = sym_tuple_length(tuple_st); + int tuple_length = ref_tuple_length(tuple_st); if (tuple_length == -1) { // Unknown length - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } else { assert(index < tuple_length); - res = sym_tuple_getitem(ctx, tuple_st, index); + res = ref_tuple_getitem(ctx, tuple_st, index); } } else { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } } op(_TO_BOOL, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); + res = ref_new_truthiness(ctx, value, true); } } op(_TO_BOOL_BOOL, (value -- value)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { - sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); + ref_set_type(value, &PyBool_Type); + value = ref_new_truthiness(ctx, value, true); } } op(_TO_BOOL_INT, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - sym_set_type(value, &PyLong_Type); - res = sym_new_truthiness(ctx, value, true); + ref_set_type(value, &PyLong_Type); + res = ref_new_truthiness(ctx, value, true); } } op(_TO_BOOL_LIST, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_NONE, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - sym_set_const(value, Py_None); - res = sym_new_const(ctx, Py_False); + ref_set_const(value, Py_None); + res = ref_new_const(ctx, Py_False); } } op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyUnicode_Type)) { + if (ref_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyUnicode_Type); + ref_set_type(nos, &PyUnicode_Type); } op(_GUARD_TOS_UNICODE, (value -- value)) { - if (sym_matches_type(value, &PyUnicode_Type)) { + if (ref_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyUnicode_Type); + ref_set_type(value, &PyUnicode_Type); } op(_TO_BOOL_STR, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); + res = ref_new_truthiness(ctx, value, true); } } op(_UNARY_NOT, (value -- res)) { - sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, false); + ref_set_type(value, &PyBool_Type); + res = ref_new_truthiness(ctx, value, false); } op(_COMPARE_OP, (left, right -- res)) { if (oparg & 16) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } else { - res = _Py_uop_sym_new_not_null(ctx); + res = _Py_uop_ref_new_not_null(ctx); } } op(_COMPARE_OP_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), - sym_get_const(ctx, right), + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *tmp = PyObject_RichCompare(ref_get_const(ctx, left), + ref_get_const(ctx, right), oparg >> 5); if (tmp == NULL) { goto error; @@ -505,42 +501,42 @@ dummy_func(void) { assert(PyBool_Check(tmp)); assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = sym_new_const(ctx, tmp); + res = ref_new_const(ctx, tmp); Py_DECREF(tmp); } else { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } } op(_COMPARE_OP_FLOAT, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_STR, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } op(_IS_OP, (left, right -- b)) { - b = sym_new_type(ctx, &PyBool_Type); + b = ref_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP, (left, right -- b)) { - b = sym_new_type(ctx, &PyBool_Type); + b = ref_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP_SET, (left, right -- b)) { - b = sym_new_type(ctx, &PyBool_Type); + b = ref_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP_DICT, (left, right -- b)) { - b = sym_new_type(ctx, &PyBool_Type); + b = ref_new_type(ctx, &PyBool_Type); } op(_LOAD_CONST, (-- value)) { PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(ref_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { @@ -548,35 +544,35 @@ dummy_func(void) { assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(ref_new_const(ctx, val)); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -585,23 +581,23 @@ dummy_func(void) { } op(_SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); } op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)offset; } op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr)) { (void)dict_version; (void)index; - attr = NULL; - if (sym_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); + attr = PyJitRef_NULL; + if (ref_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)ref_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; uint64_t watched_mutations = get_mutations(dict); @@ -609,20 +605,20 @@ dummy_func(void) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + attr = ref_new_const(ctx, res); } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { /* No conversion made. We don't know what `attr` is. */ - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); } } op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); - null[0] = sym_new_null(ctx); + null[0] = ref_new_null(ctx); } else { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -631,25 +627,25 @@ dummy_func(void) { op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { (void)owner; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); if (oparg & 1) { - self_or_null[0] = sym_new_unknown(ctx); + self_or_null[0] = ref_new_unknown(ctx); } } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr)) { - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)hint; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)index; } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); + PyTypeObject *type = (PyTypeObject *)ref_get_const(ctx, owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -658,7 +654,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -667,7 +663,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -676,7 +672,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -686,7 +682,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -696,7 +692,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -711,26 +707,26 @@ dummy_func(void) { } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(sym_get_const(ctx, callable))); + if (ref_is_const(ctx, callable) && ref_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(ref_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + this_instr->operand1 = (uintptr_t)ref_get_const(ctx, callable); } - sym_set_type(callable, &PyFunction_Type); + ref_set_type(callable, &PyFunction_Type); } op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(ctx, callable)) { - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); + assert(ref_matches_type(callable, &PyFunction_Type)); + if (ref_is_const(ctx, callable)) { + if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)ref_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + if (co->co_argcount == oparg + !ref_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } } @@ -738,8 +734,8 @@ dummy_func(void) { } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + ref_set_null(null); + ref_set_type(callable, &PyMethod_Type); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -756,13 +752,13 @@ dummy_func(void) { assert(self_or_null != NULL); assert(args != NULL); - if (sym_is_not_null(self_or_null)) { + if (ref_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; } - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { new_frame = frame_new(ctx, co, 0, args, argcount); } else { new_frame = frame_new(ctx, co, 0, NULL, 0); @@ -772,8 +768,8 @@ dummy_func(void) { op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -796,8 +792,8 @@ dummy_func(void) { op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)type_version; (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { @@ -806,7 +802,7 @@ dummy_func(void) { } op(_RETURN_VALUE, (retval -- res)) { - JitOptSymbol *temp = retval; + JitOptRef temp = retval; DEAD(retval); SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; @@ -835,7 +831,7 @@ dummy_func(void) { ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); /* Stack space handling */ assert(corresponding_check_stack == NULL); @@ -853,7 +849,7 @@ dummy_func(void) { } op(_YIELD_VALUE, (unused -- value)) { - value = sym_new_unknown(ctx); + value = ref_new_unknown(ctx); } op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) { @@ -917,7 +913,7 @@ dummy_func(void) { (void)top; /* This has to be done manually */ for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); + values[i] = ref_new_unknown(ctx); } } @@ -926,40 +922,40 @@ dummy_func(void) { /* This has to be done manually */ int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); + values[i] = ref_new_unknown(ctx); } } op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { - next = sym_new_type(ctx, &PyLong_Type); + next = ref_new_type(ctx, &PyLong_Type); } op(_CALL_TYPE_1, (unused, unused, arg -- res)) { - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + if (ref_has_type(arg)) { + res = ref_new_const(ctx, (PyObject *)ref_get_type(arg)); } else { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } } op(_CALL_STR_1, (unused, unused, arg -- res)) { - if (sym_matches_type(arg, &PyUnicode_Type)) { + if (ref_matches_type(arg, &PyUnicode_Type)) { // e.g. str('foo') or str(foo) where foo is known to be a string res = arg; } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } } op(_CALL_ISINSTANCE, (unused, unused, instance, cls -- res)) { // the result is always a bool, but sometimes we can // narrow it down to True or False - res = sym_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = sym_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); - if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { + res = ref_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = ref_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)ref_get_const(ctx, cls); + if (inst_type && cls_o && ref_matches_type(cls, &PyType_Type)) { // isinstance(inst, cls) where both inst and cls have // known types, meaning we can deduce either True or False @@ -968,50 +964,50 @@ dummy_func(void) { if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { out = Py_True; } - sym_set_const(res, out); + ref_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } } op(_GUARD_IS_TRUE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + if (ref_is_const(ctx, flag)) { + PyObject *value = ref_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } - sym_set_const(flag, Py_True); + ref_set_const(flag, Py_True); } op(_GUARD_IS_FALSE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + if (ref_is_const(ctx, flag)) { + PyObject *value = ref_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } - sym_set_const(flag, Py_False); + ref_set_const(flag, Py_False); } op(_GUARD_IS_NONE_POP, (val -- )) { - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); + if (ref_is_const(ctx, val)) { + PyObject *value = ref_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); + else if (ref_has_type(val)) { + assert(!ref_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(val, Py_None); + ref_set_const(val, Py_None); } op(_GUARD_IS_NOT_NONE_POP, (val -- )) { - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); + if (ref_is_const(ctx, val)) { + PyObject *value = ref_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); + else if (ref_has_type(val)) { + assert(!ref_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } } @@ -1025,13 +1021,13 @@ dummy_func(void) { } op(_INSERT_NULL, (self -- method_and_self[2])) { - method_and_self[0] = sym_new_null(ctx); + method_and_self[0] = ref_new_null(ctx); method_and_self[1] = self; } op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { - method_and_self[0] = sym_new_not_null(ctx); - method_and_self[1] = sym_new_unknown(ctx); + method_and_self[0] = ref_new_not_null(ctx); + method_and_self[1] = ref_new_unknown(ctx); } op(_JUMP_TO_TOP, (--)) { @@ -1045,154 +1041,154 @@ dummy_func(void) { op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - res = sym_new_const(ctx, Py_True); + res = ref_new_const(ctx, Py_True); } op(_BUILD_TUPLE, (values[oparg] -- tup)) { - tup = sym_new_tuple(ctx, oparg, values); + tup = ref_new_tuple(ctx, oparg, values); } op(_BUILD_LIST, (values[oparg] -- list)) { - list = sym_new_type(ctx, &PyList_Type); + list = ref_new_type(ctx, &PyList_Type); } op(_BUILD_SLICE, (args[oparg] -- slice)) { - slice = sym_new_type(ctx, &PySlice_Type); + slice = ref_new_type(ctx, &PySlice_Type); } op(_BUILD_MAP, (values[oparg*2] -- map)) { - map = sym_new_type(ctx, &PyDict_Type); + map = ref_new_type(ctx, &PyDict_Type); } op(_BUILD_STRING, (pieces[oparg] -- str)) { - str = sym_new_type(ctx, &PyUnicode_Type); + str = ref_new_type(ctx, &PyUnicode_Type); } op(_BUILD_SET, (values[oparg] -- set)) { - set = sym_new_type(ctx, &PySet_Type); + set = ref_new_type(ctx, &PySet_Type); } op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { - val0 = sym_tuple_getitem(ctx, seq, 0); - val1 = sym_tuple_getitem(ctx, seq, 1); + val0 = ref_tuple_getitem(ctx, seq, 0); + val1 = ref_tuple_getitem(ctx, seq, 1); } op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { for (int i = 0; i < oparg; i++) { - values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); + values[i] = ref_tuple_getitem(ctx, seq, oparg - i - 1); } } op(_CALL_TUPLE_1, (callable, null, arg -- res)) { - if (sym_matches_type(arg, &PyTuple_Type)) { + if (ref_matches_type(arg, &PyTuple_Type)) { // e.g. tuple((1, 2)) or tuple(foo) where foo is known to be a tuple res = arg; } else { - res = sym_new_type(ctx, &PyTuple_Type); + res = ref_new_type(ctx, &PyTuple_Type); } } op(_GUARD_TOS_LIST, (tos -- tos)) { - if (sym_matches_type(tos, &PyList_Type)) { + if (ref_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyList_Type); + ref_set_type(tos, &PyList_Type); } op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyList_Type)) { + if (ref_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyList_Type); + ref_set_type(nos, &PyList_Type); } op(_GUARD_TOS_TUPLE, (tos -- tos)) { - if (sym_matches_type(tos, &PyTuple_Type)) { + if (ref_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyTuple_Type); + ref_set_type(tos, &PyTuple_Type); } op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyTuple_Type)) { + if (ref_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyTuple_Type); + ref_set_type(nos, &PyTuple_Type); } op(_GUARD_TOS_DICT, (tos -- tos)) { - if (sym_matches_type(tos, &PyDict_Type)) { + if (ref_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyDict_Type); + ref_set_type(tos, &PyDict_Type); } op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyDict_Type)) { + if (ref_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyDict_Type); + ref_set_type(nos, &PyDict_Type); } op(_GUARD_TOS_ANY_SET, (tos -- tos)) { - if (sym_matches_type(tos, &PySet_Type) || - sym_matches_type(tos, &PyFrozenSet_Type)) + if (ref_matches_type(tos, &PySet_Type) || + ref_matches_type(tos, &PyFrozenSet_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } } op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { - if (sym_is_null(null)) { + if (ref_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_null(null); + ref_set_null(null); } op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { - if (sym_is_not_null(nos)) { + if (ref_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_non_null(nos); + ref_set_non_null(nos); } op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { - if (sym_is_null(null)) { + if (ref_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_null(null); + ref_set_null(null); } op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyType_Type); + ref_set_const(callable, (PyObject *)&PyType_Type); } op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyTuple_Type); + ref_set_const(callable, (PyObject *)&PyTuple_Type); } op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { - if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyUnicode_Type); + ref_set_const(callable, (PyObject *)&PyUnicode_Type); } op(_CALL_LEN, (unused, unused, unused -- res)) { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } op(_GET_LEN, (obj -- obj, len)) { - int tuple_length = sym_tuple_length(obj); + int tuple_length = ref_tuple_length(obj); if (tuple_length == -1) { - len = sym_new_type(ctx, &PyLong_Type); + len = ref_new_type(ctx, &PyLong_Type); } else { assert(tuple_length >= 0); @@ -1203,33 +1199,33 @@ dummy_func(void) { if (_Py_IsImmortal(temp)) { REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - len = sym_new_const(ctx, temp); + len = ref_new_const(ctx, temp); Py_DECREF(temp); } } op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (sym_get_const(ctx, callable) == len) { + if (ref_get_const(ctx, callable) == len) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, len); + ref_set_const(callable, len); } op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (sym_get_const(ctx, callable) == isinstance) { + if (ref_get_const(ctx, callable) == isinstance) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, isinstance); + ref_set_const(callable, isinstance); } op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (sym_get_const(ctx, callable) == list_append) { + if (ref_get_const(ctx, callable) == list_append) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, list_append); + ref_set_const(callable, list_append); } // END BYTECODES // diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index e6d191889b4476..f59a9cbd6d217a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -26,12 +26,11 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - if (sym_is_null(value)) { + if (ref_is_null(value)) { ctx->done = true; } - sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -39,9 +38,8 @@ } case _LOAD_FAST: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -49,9 +47,8 @@ } case _LOAD_FAST_BORROW: { - JitOptSymbol *value; - value = GETLOCAL(oparg); - sym_set_skip_refcount(ctx, value); + JitOptRef value; + value = PyJitRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -59,11 +56,10 @@ } case _LOAD_FAST_AND_CLEAR: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = ref_new_null(ctx); GETLOCAL(oparg) = temp; - sym_set_dont_skip_refcount(ctx, value); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -71,10 +67,10 @@ } case _LOAD_CONST: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(ref_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -82,12 +78,12 @@ } case _LOAD_SMALL_INT: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(ref_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -95,7 +91,7 @@ } case _STORE_FAST: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -116,8 +112,8 @@ } case _PUSH_NULL: { - JitOptSymbol *res; - res = sym_new_null(ctx); + JitOptRef res; + res = ref_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -137,8 +133,8 @@ } case _END_SEND: { - JitOptSymbol *val; - val = sym_new_not_null(ctx); + JitOptRef val; + val = ref_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -146,76 +142,76 @@ } case _UNARY_NEGATIVE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; - sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, false); + ref_set_type(value, &PyBool_Type); + res = ref_new_truthiness(ctx, value, false); stack_pointer[-1] = res; break; } case _TO_BOOL: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); + res = ref_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { - sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); + ref_set_type(value, &PyBool_Type); + value = ref_new_truthiness(ctx, value, true); } stack_pointer[-1] = value; break; } case _TO_BOOL_INT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - sym_set_type(value, &PyLong_Type); - res = sym_new_truthiness(ctx, value, true); + ref_set_type(value, &PyLong_Type); + res = ref_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; } case _GUARD_NOS_LIST: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyList_Type)) { + if (ref_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyList_Type); + ref_set_type(nos, &PyList_Type); break; } case _GUARD_TOS_LIST: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyList_Type)) { + if (ref_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyList_Type); + ref_set_type(tos, &PyList_Type); break; } @@ -224,119 +220,119 @@ } case _TO_BOOL_LIST: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - sym_set_const(value, Py_None); - res = sym_new_const(ctx, Py_False); + ref_set_const(value, Py_None); + res = ref_new_const(ctx, Py_False); } stack_pointer[-1] = res; break; } case _GUARD_NOS_UNICODE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyUnicode_Type)) { + if (ref_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyUnicode_Type); + ref_set_type(nos, &PyUnicode_Type); break; } case _GUARD_TOS_UNICODE: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyUnicode_Type)) { + if (ref_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyUnicode_Type); + ref_set_type(value, &PyUnicode_Type); break; } case _TO_BOOL_STR: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = sym_new_truthiness(ctx, value, true); + res = ref_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; } case _REPLACE_WITH_TRUE: { - JitOptSymbol *res; + JitOptRef res; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - res = sym_new_const(ctx, Py_True); + res = ref_new_const(ctx, Py_True); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GUARD_NOS_INT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type)) { + if (ref_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); + ref_set_type(left, &PyLong_Type); break; } case _GUARD_TOS_INT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyLong_Type)) { + if (ref_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyLong_Type); + ref_set_type(value, &PyLong_Type); break; } case _BINARY_OP_MULTIPLY_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -344,27 +340,27 @@ } case _BINARY_OP_ADD_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -372,27 +368,27 @@ } case _BINARY_OP_SUBTRACT_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)ref_get_const(ctx, left), + (PyLongObject *)ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -400,51 +396,51 @@ } case _GUARD_NOS_FLOAT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyFloat_Type)) { + if (ref_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyFloat_Type); + ref_set_type(left, &PyFloat_Type); break; } case _GUARD_TOS_FLOAT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyFloat_Type)) { + if (ref_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(value, &PyFloat_Type); + ref_set_type(value, &PyFloat_Type); break; } case _BINARY_OP_MULTIPLY_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -452,31 +448,31 @@ } case _BINARY_OP_ADD_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -484,31 +480,31 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyFloat_CheckExact(ref_get_const(ctx, left))); + assert(PyFloat_CheckExact(ref_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); + PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } - if (sym_is_skip_refcount(ctx, left) && sym_is_skip_refcount(ctx, right)) { + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } stack_pointer[-1] = res; @@ -516,8 +512,8 @@ } case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -525,8 +521,8 @@ } case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -534,8 +530,8 @@ } case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -543,26 +539,26 @@ } case _BINARY_OP_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); + assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -570,23 +566,23 @@ } case _BINARY_OP_INPLACE_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; + JitOptRef right; + JitOptRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - JitOptSymbol *res; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); + JitOptRef res; + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); + assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res = ref_new_const(ctx, temp); Py_DECREF(temp); } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } GETLOCAL(this_instr->operand0) = res; stack_pointer += -2; @@ -599,8 +595,8 @@ } case _BINARY_OP_EXTEND: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -608,8 +604,8 @@ } case _BINARY_SLICE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -623,8 +619,8 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -632,8 +628,8 @@ } case _BINARY_OP_SUBSCR_LIST_SLICE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -641,8 +637,8 @@ } case _BINARY_OP_SUBSCR_STR_INT: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyUnicode_Type); + JitOptRef res; + res = ref_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -650,47 +646,47 @@ } case _GUARD_NOS_TUPLE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyTuple_Type)) { + if (ref_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyTuple_Type); + ref_set_type(nos, &PyTuple_Type); break; } case _GUARD_TOS_TUPLE: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyTuple_Type)) { + if (ref_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyTuple_Type); + ref_set_type(tos, &PyTuple_Type); break; } case _BINARY_OP_SUBSCR_TUPLE_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *tuple_st; - JitOptSymbol *res; + JitOptRef sub_st; + JitOptRef tuple_st; + JitOptRef res; sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; - assert(sym_matches_type(tuple_st, &PyTuple_Type)); - if (sym_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); - long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); + assert(ref_matches_type(tuple_st, &PyTuple_Type)); + if (ref_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(ref_get_const(ctx, sub_st))); + long index = PyLong_AsLong(ref_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = sym_tuple_length(tuple_st); + int tuple_length = ref_tuple_length(tuple_st); if (tuple_length == -1) { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } else { assert(index < tuple_length); - res = sym_tuple_getitem(ctx, tuple_st, index); + res = ref_tuple_getitem(ctx, tuple_st, index); } } else { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -699,28 +695,28 @@ } case _GUARD_NOS_DICT: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyDict_Type)) { + if (ref_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyDict_Type); + ref_set_type(nos, &PyDict_Type); break; } case _GUARD_TOS_DICT: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyDict_Type)) { + if (ref_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyDict_Type); + ref_set_type(tos, &PyDict_Type); break; } case _BINARY_OP_SUBSCR_DICT: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -728,8 +724,8 @@ } case _BINARY_OP_SUBSCR_CHECK_FUNC: { - JitOptSymbol *getitem; - getitem = sym_new_not_null(ctx); + JitOptRef getitem; + getitem = ref_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -740,7 +736,7 @@ _Py_UOpsAbstractFrame *new_frame; new_frame = NULL; ctx->done = true; - stack_pointer[-3] = (JitOptSymbol *)new_frame; + stack_pointer[-3].bits = (uintptr_t)new_frame; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -783,15 +779,15 @@ } case _CALL_INTRINSIC_1: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -799,10 +795,10 @@ } case _RETURN_VALUE: { - JitOptSymbol *retval; - JitOptSymbol *res; + JitOptRef retval; + JitOptRef res; retval = stack_pointer[-1]; - JitOptSymbol *temp = retval; + JitOptRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -826,15 +822,15 @@ } case _GET_AITER: { - JitOptSymbol *iter; - iter = sym_new_not_null(ctx); + JitOptRef iter; + iter = ref_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - JitOptSymbol *awaitable; - awaitable = sym_new_not_null(ctx); + JitOptRef awaitable; + awaitable = ref_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -842,8 +838,8 @@ } case _GET_AWAITABLE: { - JitOptSymbol *iter; - iter = sym_new_not_null(ctx); + JitOptRef iter; + iter = ref_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -854,13 +850,13 @@ _Py_UOpsAbstractFrame *gen_frame; gen_frame = NULL; ctx->done = true; - stack_pointer[-1] = (JitOptSymbol *)gen_frame; + stack_pointer[-1].bits = (uintptr_t)gen_frame; break; } case _YIELD_VALUE: { - JitOptSymbol *value; - value = sym_new_unknown(ctx); + JitOptRef value; + value = ref_new_unknown(ctx); stack_pointer[-1] = value; break; } @@ -872,8 +868,8 @@ } case _LOAD_COMMON_CONSTANT: { - JitOptSymbol *value; - value = sym_new_not_null(ctx); + JitOptRef value; + value = ref_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -881,8 +877,8 @@ } case _LOAD_BUILD_CLASS: { - JitOptSymbol *bc; - bc = sym_new_not_null(ctx); + JitOptRef bc; + bc = ref_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -900,13 +896,13 @@ } case _UNPACK_SEQUENCE: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[-1 + oparg]; (void)top; for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); + values[i] = ref_new_unknown(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -914,12 +910,12 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol *val1; - JitOptSymbol *val0; + JitOptRef seq; + JitOptRef val1; + JitOptRef val0; seq = stack_pointer[-1]; - val0 = sym_tuple_getitem(ctx, seq, 0); - val1 = sym_tuple_getitem(ctx, seq, 1); + val0 = ref_tuple_getitem(ctx, seq, 0); + val1 = ref_tuple_getitem(ctx, seq, 1); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -928,12 +924,12 @@ } case _UNPACK_SEQUENCE_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol **values; + JitOptRef seq; + JitOptRef *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { - values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); + values[i] = ref_tuple_getitem(ctx, seq, oparg - i - 1); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -941,10 +937,10 @@ } case _UNPACK_SEQUENCE_LIST: { - JitOptSymbol **values; + JitOptRef *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_not_null(ctx); + values[_i] = ref_new_not_null(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -952,14 +948,14 @@ } case _UNPACK_EX: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; (void)top; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); + values[i] = ref_new_unknown(ctx); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -989,8 +985,8 @@ } case _LOAD_LOCALS: { - JitOptSymbol *locals; - locals = sym_new_not_null(ctx); + JitOptRef locals; + locals = ref_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1000,8 +996,8 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - JitOptSymbol *v; - v = sym_new_not_null(ctx); + JitOptRef v; + v = ref_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1009,20 +1005,20 @@ } case _LOAD_GLOBAL: { - JitOptSymbol **res; + JitOptRef *res; res = &stack_pointer[0]; - res[0] = sym_new_not_null(ctx); + res[0] = ref_new_not_null(ctx); stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_NULL_CONDITIONAL: { - JitOptSymbol **null; + JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); - null[0] = sym_new_null(ctx); + null[0] = ref_new_null(ctx); } else { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1037,8 +1033,8 @@ } case _LOAD_GLOBAL_MODULE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1046,8 +1042,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1067,15 +1063,15 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - JitOptSymbol *value; - value = sym_new_not_null(ctx); + JitOptRef value; + value = ref_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - JitOptSymbol *value; - value = sym_new_not_null(ctx); + JitOptRef value; + value = ref_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1093,8 +1089,8 @@ } case _BUILD_STRING: { - JitOptSymbol *str; - str = sym_new_type(ctx, &PyUnicode_Type); + JitOptRef str; + str = ref_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1102,8 +1098,8 @@ } case _BUILD_INTERPOLATION: { - JitOptSymbol *interpolation; - interpolation = sym_new_not_null(ctx); + JitOptRef interpolation; + interpolation = ref_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -1111,8 +1107,8 @@ } case _BUILD_TEMPLATE: { - JitOptSymbol *template; - template = sym_new_not_null(ctx); + JitOptRef template; + template = ref_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1120,10 +1116,10 @@ } case _BUILD_TUPLE: { - JitOptSymbol **values; - JitOptSymbol *tup; + JitOptRef *values; + JitOptRef tup; values = &stack_pointer[-oparg]; - tup = sym_new_tuple(ctx, oparg, values); + tup = ref_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1131,8 +1127,8 @@ } case _BUILD_LIST: { - JitOptSymbol *list; - list = sym_new_type(ctx, &PyList_Type); + JitOptRef list; + list = ref_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1152,8 +1148,8 @@ } case _BUILD_SET: { - JitOptSymbol *set; - set = sym_new_type(ctx, &PySet_Type); + JitOptRef set; + set = ref_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1161,8 +1157,8 @@ } case _BUILD_MAP: { - JitOptSymbol *map; - map = sym_new_type(ctx, &PyDict_Type); + JitOptRef map; + map = ref_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); @@ -1192,8 +1188,8 @@ } case _LOAD_SUPER_ATTR_ATTR: { - JitOptSymbol *attr_st; - attr_st = sym_new_not_null(ctx); + JitOptRef attr_st; + attr_st = ref_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1201,10 +1197,10 @@ } case _LOAD_SUPER_ATTR_METHOD: { - JitOptSymbol *attr; - JitOptSymbol *self_or_null; - attr = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + JitOptRef attr; + JitOptRef self_or_null; + attr = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1213,15 +1209,15 @@ } case _LOAD_ATTR: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol **self_or_null; + JitOptRef owner; + JitOptRef attr; + JitOptRef *self_or_null; owner = stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); if (oparg & 1) { - self_or_null[0] = sym_new_unknown(ctx); + self_or_null[0] = ref_new_unknown(ctx); } stack_pointer[-1] = attr; stack_pointer += (oparg&1); @@ -1230,16 +1226,16 @@ } case _GUARD_TYPE_VERSION: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); - if (sym_matches_type_version(owner, type_version)) { + if (ref_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { PyTypeObject *type = _PyType_LookupByVersion(type_version); if (type) { - if (sym_set_type_version(owner, type_version)) { + if (ref_set_type_version(owner, type_version)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); } @@ -1257,25 +1253,25 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t offset = (uint16_t)this_instr->operand0; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)offset; stack_pointer[-1] = attr; break; } case _LOAD_ATTR_MODULE: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand0; uint16_t index = (uint16_t)this_instr->operand0; (void)dict_version; (void)index; - attr = NULL; - if (sym_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); + attr = PyJitRef_NULL; + if (ref_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)ref_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; stack_pointer[-1] = attr; @@ -1284,58 +1280,58 @@ PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + attr = ref_new_const(ctx, res); } } } - if (attr == NULL) { - attr = sym_new_not_null(ctx); + if (PyJitRef_IsNull(attr)) { + attr = ref_new_not_null(ctx); } stack_pointer[-1] = attr; break; } case _LOAD_ATTR_WITH_HINT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t hint = (uint16_t)this_instr->operand0; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)hint; stack_pointer[-1] = attr; break; } case _LOAD_ATTR_SLOT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t index = (uint16_t)this_instr->operand0; - attr = sym_new_not_null(ctx); + attr = ref_new_not_null(ctx); (void)index; stack_pointer[-1] = attr; break; } case _CHECK_ATTR_CLASS: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { - if (type == sym_get_const(ctx, owner)) { + if (type == ref_get_const(ctx, owner)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { - sym_set_const(owner, type); + ref_set_const(owner, type); } } break; } case _LOAD_ATTR_CLASS: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); + PyTypeObject *type = (PyTypeObject *)ref_get_const(ctx, owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1350,7 +1346,7 @@ (void)fget; new_frame = NULL; ctx->done = true; - stack_pointer[-1] = (JitOptSymbol *)new_frame; + stack_pointer[-1].bits = (uintptr_t)new_frame; break; } @@ -1379,12 +1375,12 @@ } case _COMPARE_OP: { - JitOptSymbol *res; + JitOptRef res; if (oparg & 16) { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); } else { - res = _Py_uop_sym_new_not_null(ctx); + res = _Py_uop_ref_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1393,8 +1389,8 @@ } case _COMPARE_OP_FLOAT: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); + JitOptRef res; + res = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1402,16 +1398,16 @@ } case _COMPARE_OP_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), - sym_get_const(ctx, right), + if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { + assert(PyLong_CheckExact(ref_get_const(ctx, left))); + assert(PyLong_CheckExact(ref_get_const(ctx, right))); + PyObject *tmp = PyObject_RichCompare(ref_get_const(ctx, left), + ref_get_const(ctx, right), oparg >> 5); if (tmp == NULL) { goto error; @@ -1419,14 +1415,14 @@ assert(PyBool_Check(tmp)); assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = sym_new_const(ctx, tmp); + res = ref_new_const(ctx, tmp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(tmp); } else { - res = sym_new_type(ctx, &PyBool_Type); + res = ref_new_type(ctx, &PyBool_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -1434,8 +1430,8 @@ } case _COMPARE_OP_STR: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); + JitOptRef res; + res = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1443,8 +1439,8 @@ } case _IS_OP: { - JitOptSymbol *b; - b = sym_new_type(ctx, &PyBool_Type); + JitOptRef b; + b = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1452,8 +1448,8 @@ } case _CONTAINS_OP: { - JitOptSymbol *b; - b = sym_new_type(ctx, &PyBool_Type); + JitOptRef b; + b = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1461,10 +1457,10 @@ } case _GUARD_TOS_ANY_SET: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PySet_Type) || - sym_matches_type(tos, &PyFrozenSet_Type)) + if (ref_matches_type(tos, &PySet_Type) || + ref_matches_type(tos, &PyFrozenSet_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } @@ -1472,8 +1468,8 @@ } case _CONTAINS_OP_SET: { - JitOptSymbol *b; - b = sym_new_type(ctx, &PyBool_Type); + JitOptRef b; + b = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1481,8 +1477,8 @@ } case _CONTAINS_OP_DICT: { - JitOptSymbol *b; - b = sym_new_type(ctx, &PyBool_Type); + JitOptRef b; + b = ref_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1490,25 +1486,25 @@ } case _CHECK_EG_MATCH: { - JitOptSymbol *rest; - JitOptSymbol *match; - rest = sym_new_not_null(ctx); - match = sym_new_not_null(ctx); + JitOptRef rest; + JitOptRef match; + rest = ref_new_not_null(ctx); + match = ref_new_not_null(ctx); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; } case _CHECK_EXC_MATCH: { - JitOptSymbol *b; - b = sym_new_not_null(ctx); + JitOptRef b; + b = ref_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1516,8 +1512,8 @@ } case _IMPORT_FROM: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1529,19 +1525,19 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - JitOptSymbol *b; - b = sym_new_not_null(ctx); + JitOptRef b; + b = ref_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - JitOptSymbol *obj; - JitOptSymbol *len; + JitOptRef obj; + JitOptRef len; obj = stack_pointer[-1]; - int tuple_length = sym_tuple_length(obj); + int tuple_length = ref_tuple_length(obj); if (tuple_length == -1) { - len = sym_new_type(ctx, &PyLong_Type); + len = ref_new_type(ctx, &PyLong_Type); } else { assert(tuple_length >= 0); @@ -1552,7 +1548,7 @@ if (_Py_IsImmortal(temp)) { REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - len = sym_new_const(ctx, temp); + len = ref_new_const(ctx, temp); stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1566,8 +1562,8 @@ } case _MATCH_CLASS: { - JitOptSymbol *attrs; - attrs = sym_new_not_null(ctx); + JitOptRef attrs; + attrs = ref_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1575,8 +1571,8 @@ } case _MATCH_MAPPING: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1584,8 +1580,8 @@ } case _MATCH_SEQUENCE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1593,8 +1589,8 @@ } case _MATCH_KEYS: { - JitOptSymbol *values_or_none; - values_or_none = sym_new_not_null(ctx); + JitOptRef values_or_none; + values_or_none = ref_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1602,10 +1598,10 @@ } case _GET_ITER: { - JitOptSymbol *iter; - JitOptSymbol *index_or_null; - iter = sym_new_not_null(ctx); - index_or_null = sym_new_not_null(ctx); + JitOptRef iter; + JitOptRef index_or_null; + iter = ref_new_not_null(ctx); + index_or_null = ref_new_not_null(ctx); stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; @@ -1614,8 +1610,8 @@ } case _GET_YIELD_FROM_ITER: { - JitOptSymbol *iter; - iter = sym_new_not_null(ctx); + JitOptRef iter; + iter = ref_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -1623,8 +1619,8 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - JitOptSymbol *next; - next = sym_new_not_null(ctx); + JitOptRef next; + next = ref_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1646,8 +1642,8 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ case _ITER_NEXT_LIST_TIER_TWO: { - JitOptSymbol *next; - next = sym_new_not_null(ctx); + JitOptRef next; + next = ref_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1665,8 +1661,8 @@ } case _ITER_NEXT_TUPLE: { - JitOptSymbol *next; - next = sym_new_not_null(ctx); + JitOptRef next; + next = ref_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1684,8 +1680,8 @@ } case _ITER_NEXT_RANGE: { - JitOptSymbol *next; - next = sym_new_type(ctx, &PyLong_Type); + JitOptRef next; + next = ref_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1696,18 +1692,18 @@ _Py_UOpsAbstractFrame *gen_frame; gen_frame = NULL; ctx->done = true; - stack_pointer[0] = (JitOptSymbol *)gen_frame; + stack_pointer[0].bits = (uintptr_t)gen_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _INSERT_NULL: { - JitOptSymbol *self; - JitOptSymbol **method_and_self; + JitOptRef self; + JitOptRef *method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; - method_and_self[0] = sym_new_null(ctx); + method_and_self[0] = ref_new_null(ctx); method_and_self[1] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1715,16 +1711,16 @@ } case _LOAD_SPECIAL: { - JitOptSymbol **method_and_self; + JitOptRef *method_and_self; method_and_self = &stack_pointer[-2]; - method_and_self[0] = sym_new_not_null(ctx); - method_and_self[1] = sym_new_unknown(ctx); + method_and_self[0] = ref_new_not_null(ctx); + method_and_self[1] = ref_new_unknown(ctx); break; } case _WITH_EXCEPT_START: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1732,10 +1728,10 @@ } case _PUSH_EXC_INFO: { - JitOptSymbol *prev_exc; - JitOptSymbol *new_exc; - prev_exc = sym_new_not_null(ctx); - new_exc = sym_new_not_null(ctx); + JitOptRef prev_exc; + JitOptRef new_exc; + prev_exc = ref_new_not_null(ctx); + new_exc = ref_new_not_null(ctx); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -1752,13 +1748,13 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1772,13 +1768,13 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1792,12 +1788,12 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1807,12 +1803,12 @@ } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1826,13 +1822,13 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = sym_get_type(owner); + PyTypeObject *type = ref_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1846,15 +1842,15 @@ } case _MAYBE_EXPAND_METHOD: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -1874,22 +1870,22 @@ break; } new_frame = frame_new(ctx, co, 0, NULL, 0); - stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _CHECK_FUNCTION_VERSION: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; - if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(sym_get_const(ctx, callable))); + if (ref_is_const(ctx, callable) && ref_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(ref_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + this_instr->operand1 = (uintptr_t)ref_get_const(ctx, callable); } - sym_set_type(callable, &PyFunction_Type); + ref_set_type(callable, &PyFunction_Type); break; } @@ -1910,8 +1906,8 @@ } case _CALL_NON_PY_GENERAL: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1919,22 +1915,22 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *null; - JitOptSymbol *callable; + JitOptRef null; + JitOptRef callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + ref_set_null(null); + ref_set_type(callable, &PyMethod_Type); break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -1948,16 +1944,16 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(ctx, callable)) { - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); + assert(ref_matches_type(callable, &PyFunction_Type)); + if (ref_is_const(ctx, callable)) { + if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)ref_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + if (co->co_argcount == oparg + !ref_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } } @@ -1976,8 +1972,8 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; + JitOptRef *args; + JitOptRef self_or_null; _Py_UOpsAbstractFrame *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -1991,16 +1987,16 @@ } assert(self_or_null != NULL); assert(args != NULL); - if (sym_is_not_null(self_or_null)) { + if (ref_is_not_null(self_or_null)) { args--; argcount++; } - if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { new_frame = frame_new(ctx, co, 0, args, argcount); } else { new_frame = frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2008,7 +2004,7 @@ case _PUSH_FRAME: { _Py_UOpsAbstractFrame *new_frame; - new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1].bits; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -2039,54 +2035,54 @@ } case _GUARD_NOS_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-2]; - if (sym_is_null(null)) { + if (ref_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_null(null); + ref_set_null(null); break; } case _GUARD_NOS_NOT_NULL: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; - if (sym_is_not_null(nos)) { + if (ref_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_non_null(nos); + ref_set_non_null(nos); break; } case _GUARD_THIRD_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-3]; - if (sym_is_null(null)) { + if (ref_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_null(null); + ref_set_null(null); break; } case _GUARD_CALLABLE_TYPE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; - if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyType_Type); + ref_set_const(callable, (PyObject *)&PyType_Type); break; } case _CALL_TYPE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + if (ref_has_type(arg)) { + res = ref_new_const(ctx, (PyObject *)ref_get_type(arg)); } else { - res = sym_new_not_null(ctx); + res = ref_new_not_null(ctx); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2095,24 +2091,24 @@ } case _GUARD_CALLABLE_STR_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; - if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyUnicode_Type); + ref_set_const(callable, (PyObject *)&PyUnicode_Type); break; } case _CALL_STR_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; - if (sym_matches_type(arg, &PyUnicode_Type)) { + if (ref_matches_type(arg, &PyUnicode_Type)) { res = arg; } else { - res = sym_new_type(ctx, &PyUnicode_Type); + res = ref_new_type(ctx, &PyUnicode_Type); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2121,24 +2117,24 @@ } case _GUARD_CALLABLE_TUPLE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; - if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + if (ref_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, (PyObject *)&PyTuple_Type); + ref_set_const(callable, (PyObject *)&PyTuple_Type); break; } case _CALL_TUPLE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; - if (sym_matches_type(arg, &PyTuple_Type)) { + if (ref_matches_type(arg, &PyTuple_Type)) { res = arg; } else { - res = sym_new_type(ctx, &PyTuple_Type); + res = ref_new_type(ctx, &PyTuple_Type); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2147,17 +2143,17 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; (void)args; - callable = sym_new_not_null(ctx); - self_or_null = sym_new_not_null(ctx); + callable = ref_new_not_null(ctx); + self_or_null = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -2167,7 +2163,7 @@ _Py_UOpsAbstractFrame *init_frame; init_frame = NULL; ctx->done = true; - stack_pointer[-2 - oparg] = (JitOptSymbol *)init_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2180,8 +2176,8 @@ } case _CALL_BUILTIN_CLASS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2189,8 +2185,8 @@ } case _CALL_BUILTIN_O: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2198,8 +2194,8 @@ } case _CALL_BUILTIN_FAST: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2207,8 +2203,8 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2216,19 +2212,19 @@ } case _GUARD_CALLABLE_LEN: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (sym_get_const(ctx, callable) == len) { + if (ref_get_const(ctx, callable) == len) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, len); + ref_set_const(callable, len); break; } case _CALL_LEN: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyLong_Type); + JitOptRef res; + res = ref_new_type(ctx, &PyLong_Type); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2236,31 +2232,31 @@ } case _GUARD_CALLABLE_ISINSTANCE: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (sym_get_const(ctx, callable) == isinstance) { + if (ref_get_const(ctx, callable) == isinstance) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, isinstance); + ref_set_const(callable, isinstance); break; } case _CALL_ISINSTANCE: { - JitOptSymbol *cls; - JitOptSymbol *instance; - JitOptSymbol *res; + JitOptRef cls; + JitOptRef instance; + JitOptRef res; cls = stack_pointer[-1]; instance = stack_pointer[-2]; - res = sym_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = sym_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); - if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { + res = ref_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = ref_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)ref_get_const(ctx, cls); + if (inst_type && cls_o && ref_matches_type(cls, &PyType_Type)) { PyObject *out = Py_False; if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { out = Py_True; } - sym_set_const(res, out); + ref_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } stack_pointer[-4] = res; @@ -2270,13 +2266,13 @@ } case _GUARD_CALLABLE_LIST_APPEND: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (sym_get_const(ctx, callable) == list_append) { + if (ref_get_const(ctx, callable) == list_append) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_const(callable, list_append); + ref_set_const(callable, list_append); break; } @@ -2287,8 +2283,8 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2296,8 +2292,8 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2305,8 +2301,8 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2314,8 +2310,8 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2334,7 +2330,7 @@ _Py_UOpsAbstractFrame *new_frame; new_frame = NULL; ctx->done = true; - stack_pointer[-3 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2357,8 +2353,8 @@ } case _CALL_KW_NON_PY: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2372,15 +2368,15 @@ /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - JitOptSymbol *func; - func = sym_new_not_null(ctx); + JitOptRef func; + func = ref_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - JitOptSymbol *func_out; - func_out = sym_new_not_null(ctx); + JitOptRef func_out; + func_out = ref_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2388,11 +2384,11 @@ } case _RETURN_GENERATOR: { - JitOptSymbol *res; + JitOptRef res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); assert(corresponding_check_stack == NULL); assert(co != NULL); int framesize = co->co_framesize; @@ -2410,8 +2406,8 @@ } case _BUILD_SLICE: { - JitOptSymbol *slice; - slice = sym_new_type(ctx, &PySlice_Type); + JitOptRef slice; + slice = ref_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2419,22 +2415,22 @@ } case _CONVERT_VALUE: { - JitOptSymbol *result; - result = sym_new_not_null(ctx); + JitOptRef result; + result = ref_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + res = ref_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2442,8 +2438,8 @@ } case _COPY: { - JitOptSymbol *bottom; - JitOptSymbol *top; + JitOptRef bottom; + JitOptRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2454,43 +2450,43 @@ } case _BINARY_OP: { - JitOptSymbol *rhs; - JitOptSymbol *lhs; - JitOptSymbol *res; + JitOptRef rhs; + JitOptRef lhs; + JitOptRef res; rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; - bool lhs_int = sym_matches_type(lhs, &PyLong_Type); - bool rhs_int = sym_matches_type(rhs, &PyLong_Type); - bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); - bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); + bool lhs_int = ref_matches_type(lhs, &PyLong_Type); + bool rhs_int = ref_matches_type(rhs, &PyLong_Type); + bool lhs_float = ref_matches_type(lhs, &PyFloat_Type); + bool rhs_float = ref_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { if (rhs_float) { - res = sym_new_unknown(ctx); + res = ref_new_unknown(ctx); } else if (lhs_float) { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, rhs)) { - res = sym_new_unknown(ctx); + else if (!ref_is_const(ctx, rhs)) { + res = ref_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { - res = sym_new_type(ctx, &PyFloat_Type); + else if (_PyLong_IsNegative((PyLongObject *)ref_get_const(ctx, rhs))) { + res = ref_new_type(ctx, &PyFloat_Type); } else { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } } else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } else if (lhs_int && rhs_int) { - res = sym_new_type(ctx, &PyLong_Type); + res = ref_new_type(ctx, &PyLong_Type); } else { - res = sym_new_type(ctx, &PyFloat_Type); + res = ref_new_type(ctx, &PyFloat_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -2499,11 +2495,11 @@ } case _SWAP: { - JitOptSymbol *top; - JitOptSymbol *bottom; + JitOptRef top; + JitOptRef bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -2531,61 +2527,61 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + if (ref_is_const(ctx, flag)) { + PyObject *value = ref_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } - sym_set_const(flag, Py_True); + ref_set_const(flag, Py_True); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_FALSE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + if (ref_is_const(ctx, flag)) { + PyObject *value = ref_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } - sym_set_const(flag, Py_False); + ref_set_const(flag, Py_False); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); + if (ref_is_const(ctx, val)) { + PyObject *value = ref_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); + else if (ref_has_type(val)) { + assert(!ref_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(val, Py_None); + ref_set_const(val, Py_None); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NOT_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; - if (sym_is_const(ctx, val)) { - PyObject *value = sym_get_const(ctx, val); + if (ref_is_const(ctx, val)) { + PyObject *value = ref_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(val)) { - assert(!sym_matches_type(val, &_PyNone_Type)); + else if (ref_has_type(val)) { + assert(!ref_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } stack_pointer += -1; @@ -2625,9 +2621,9 @@ } case _LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2635,17 +2631,17 @@ } case _POP_TOP_LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2671,16 +2667,16 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _POP_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; - value = sym_new_not_null(ctx); + JitOptRef value; + value = ref_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2688,9 +2684,9 @@ } case _POP_CALL_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2698,9 +2694,9 @@ } case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2708,9 +2704,9 @@ } case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -2718,10 +2714,10 @@ } case _LOAD_CONST_UNDER_INLINE: { - JitOptSymbol *value; - JitOptSymbol *new; - value = sym_new_not_null(ctx); - new = sym_new_not_null(ctx); + JitOptRef value; + JitOptRef new; + value = ref_new_not_null(ctx); + new = ref_new_not_null(ctx); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; @@ -2730,10 +2726,10 @@ } case _LOAD_CONST_UNDER_INLINE_BORROW: { - JitOptSymbol *value; - JitOptSymbol *new; - value = sym_new_not_null(ctx); - new = sym_new_not_null(ctx); + JitOptRef value; + JitOptRef new; + value = ref_new_not_null(ctx); + new = ref_new_not_null(ctx); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 68f1ecd1d316a4..66fa66e21935a3 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -88,6 +88,12 @@ out_of_space(JitOptContext *ctx) return &NO_SPACE_SYMBOL; } +JitOptRef +out_of_space_ref(JitOptContext *ctx) +{ + return PyJitRef_FromSymbolSteal(out_of_space(ctx)); +} + static JitOptSymbol * sym_new(JitOptContext *ctx) { @@ -98,8 +104,7 @@ sym_new(JitOptContext *ctx) return NULL; } ctx->t_arena.ty_curr_number++; - self->tag = JIT_SYM_UNKNOWN_TAG; - self->skip_refcount = DONT_SKIP_REFCOUNT; + self->tag = JIT_SYM_UNKNOWN_TAG;; return self; } @@ -107,9 +112,6 @@ static void make_const(JitOptSymbol *sym, PyObject *val) { sym->tag = JIT_SYM_KNOWN_VALUE_TAG; sym->value.value = Py_NewRef(val); - // Constants don't need to be refcounted, as they are always - // kept alive by co_consts. - sym->skip_refcount = SKIP_REFCOUNT; } static inline void @@ -121,25 +123,28 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_bottom(JitOptSymbol *sym) +_Py_uop_ref_is_bottom(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_not_null(JitOptSymbol *sym) { +_Py_uop_ref_is_not_null(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return true; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_ref_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); if (truthiness < 0) { return false; } @@ -150,21 +155,22 @@ _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_null(JitOptSymbol *sym) +_Py_uop_ref_is_null(JitOptRef ref) { - return sym->tag == JIT_SYM_NULL_TAG; + return PyJitRef_AsSymbolBorrow(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * -_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return sym->value.value; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_ref_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); if (truthiness < 0) { return NULL; } @@ -176,8 +182,9 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -226,11 +233,12 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) } bool -_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version) +_Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); PyTypeObject *type = _PyType_LookupByVersion(version); if (type) { - _Py_uop_sym_set_type(ctx, sym, type); + _Py_uop_ref_set_type(ctx, ref, type); } JitSymType tag = sym->tag; switch(tag) { @@ -283,8 +291,9 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int } void -_Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val) +_Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -305,12 +314,12 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TUPLE_TAG: if (PyTuple_CheckExact(const_val)) { - Py_ssize_t len = _Py_uop_sym_tuple_length(sym); + Py_ssize_t len = _Py_uop_ref_tuple_length(ref); if (len == PyTuple_GET_SIZE(const_val)) { for (Py_ssize_t i = 0; i < len; i++) { - JitOptSymbol *sym_item = _Py_uop_sym_tuple_getitem(ctx, sym, i); + JitOptRef ref_item = _Py_uop_ref_tuple_getitem(ctx, ref, i); PyObject *item = PyTuple_GET_ITEM(const_val, i); - _Py_uop_sym_set_const(ctx, sym_item, item); + _Py_uop_ref_set_const(ctx, ref_item, item); } make_const(sym, const_val); return; @@ -333,29 +342,29 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TRUTHINESS_TAG: if (!PyBool_Check(const_val) || - (_Py_uop_sym_is_const(ctx, sym) && - _Py_uop_sym_get_const(ctx, sym) != const_val)) + (_Py_uop_ref_is_const(ctx, ref) && + _Py_uop_ref_get_const(ctx, ref) != const_val)) { sym_set_bottom(ctx, sym); return; } - JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - PyTypeObject *type = _Py_uop_sym_get_type(value); + JitOptRef value = PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->truthiness.value); + PyTypeObject *type = _Py_uop_ref_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: if (type == &PyBool_Type) { - _Py_uop_sym_set_const(ctx, value, Py_True); + _Py_uop_ref_set_const(ctx, value, Py_True); } } // value is falsey: else if (type == &PyBool_Type) { - _Py_uop_sym_set_const(ctx, value, Py_False); + _Py_uop_ref_set_const(ctx, value, Py_False); } else if (type == &PyLong_Type) { - _Py_uop_sym_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_ZERO)); + _Py_uop_ref_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_ZERO)); } else if (type == &PyUnicode_Type) { - _Py_uop_sym_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_EMPTY_STR)); + _Py_uop_ref_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_EMPTY_STR)); } // TODO: More types (GH-130415)! make_const(sym, const_val); @@ -364,8 +373,9 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val } void -_Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NULL_TAG; } @@ -375,8 +385,9 @@ _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NON_NULL_TAG; } @@ -385,85 +396,69 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } } -void -_Py_uop_sym_set_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) -{ - sym->skip_refcount = SKIP_REFCOUNT; -} - -void -_Py_uop_sym_set_dont_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) -{ - sym->skip_refcount = DONT_SKIP_REFCOUNT; -} - -bool -_Py_uop_sym_is_skip_refcount(JitOptContext *ctx, JitOptSymbol *sym) -{ - assert(sym->skip_refcount == SKIP_REFCOUNT || sym->skip_refcount == DONT_SKIP_REFCOUNT); - return sym->skip_refcount == SKIP_REFCOUNT; -} - - -JitOptSymbol * -_Py_uop_sym_new_unknown(JitOptContext *ctx) +JitOptRef +_Py_uop_ref_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - return res; + return PyJitRef_FromSymbolSteal(res); } -JitOptSymbol * -_Py_uop_sym_new_not_null(JitOptContext *ctx) +JitOptRef +_Py_uop_ref_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } res->tag = JIT_SYM_NON_NULL_TAG; - return res; + return PyJitRef_FromSymbolSteal(res); } -JitOptSymbol * -_Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) +JitOptRef +_Py_uop_ref_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_type(ctx, res, typ); - return res; + JitOptRef ref = PyJitRef_FromSymbolSteal(res); + _Py_uop_ref_set_type(ctx, ref, typ); + return ref; } // Adds a new reference to const_val, owned by the symbol. -JitOptSymbol * -_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) +JitOptRef +_Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_const(ctx, res, const_val); - return res; + JitOptRef ref = PyJitRef_FromSymbolSteal(res); + _Py_uop_ref_set_const(ctx, ref, const_val); + return ref; } -JitOptSymbol * -_Py_uop_sym_new_null(JitOptContext *ctx) +JitOptRef +_Py_uop_ref_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); if (null_sym == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_null(ctx, null_sym); - return null_sym; + JitOptRef ref = PyJitRef_FromSymbolSteal(null_sym); + _Py_uop_ref_set_null(ctx, ref); + return ref; } PyTypeObject * -_Py_uop_sym_get_type(JitOptSymbol *sym) +_Py_uop_ref_get_type(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -486,8 +481,9 @@ _Py_uop_sym_get_type(JitOptSymbol *sym) } unsigned int -_Py_uop_sym_get_type_version(JitOptSymbol *sym) +_Py_uop_ref_get_type_version(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -510,27 +506,28 @@ _Py_uop_sym_get_type_version(JitOptSymbol *sym) } bool -_Py_uop_sym_has_type(JitOptSymbol *sym) +_Py_uop_ref_has_type(JitOptRef sym) { - return _Py_uop_sym_get_type(sym) != NULL; + return _Py_uop_ref_get_type(sym) != NULL; } bool -_Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_ref_matches_type(JitOptRef sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - return _Py_uop_sym_get_type(sym) == typ; + return _Py_uop_ref_get_type(sym) == typ; } bool -_Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version) +_Py_uop_ref_matches_type_version(JitOptRef sym, unsigned int version) { - return _Py_uop_sym_get_type_version(sym) == version; + return _Py_uop_ref_get_type_version(sym) == version; } int -_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); switch(sym->tag) { case JIT_SYM_NULL_TAG: case JIT_SYM_TYPE_VERSION_TAG: @@ -550,7 +547,8 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) case JIT_SYM_TRUTHINESS_TAG: ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_ref_truthiness(ctx, + PyJitRef_FromSymbolBorrow(value)); if (truthiness < 0) { return truthiness; } @@ -576,12 +574,12 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) return -1; } -JitOptSymbol * -_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) +JitOptRef +_Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } if (size > MAX_SYMBOLIC_TUPLE_SIZE) { res->tag = JIT_SYM_KNOWN_CLASS_TAG; @@ -591,31 +589,33 @@ _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) res->tag = JIT_SYM_TUPLE_TAG; res->tuple.length = size; for (int i = 0; i < size; i++) { - res->tuple.items[i] = (uint16_t)(args[i] - allocation_base(ctx)); + res->tuple.items[i] = (uint16_t)(PyJitRef_AsSymbolBorrow(args[i]) - allocation_base(ctx)); } } - return res; + return PyJitRef_FromSymbolSteal(res); } -JitOptSymbol * -_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) +JitOptRef +_Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple) && item < PyTuple_GET_SIZE(tuple)) { - return _Py_uop_sym_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); + return _Py_uop_ref_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { - return allocation_base(ctx) + sym->tuple.items[item]; + return PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->tuple.items[item]); } - return _Py_uop_sym_new_not_null(ctx); + return _Py_uop_ref_new_not_null(ctx); } int -_Py_uop_sym_tuple_length(JitOptSymbol *sym) +_Py_uop_ref_tuple_length(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple)) { @@ -644,19 +644,27 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) return false; } -JitOptSymbol * -_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) +bool +_Py_uop_ref_is_immortal(JitOptRef ref) +{ + JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + return _Py_uop_sym_is_immortal(sym); +} + +JitOptRef +_Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { + JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref); // It's clearer to invert this in the signature: bool invert = !truthy; if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.invert == invert) { - return value; + return ref; } JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_ref_truthiness(ctx, ref); if (truthiness < 0) { res->tag = JIT_SYM_TRUTHINESS_TAG; res->truthiness.invert = invert; @@ -665,7 +673,7 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) else { make_const(res, (truthiness ^ invert) ? Py_True : Py_False); } - return res; + return PyJitRef_FromSymbolSteal(res); } // 0 on success, -1 on error. @@ -674,7 +682,7 @@ _Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); @@ -699,14 +707,14 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptSymbol *local = _Py_uop_sym_new_unknown(ctx); + JitOptRef local = _Py_uop_ref_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - JitOptSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + JitOptRef stackvar = _Py_uop_ref_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -790,45 +798,44 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - JitOptSymbol *sym = _Py_uop_sym_new_unknown(ctx); - if (sym == NULL) { + JitOptRef ref = _Py_uop_ref_new_unknown(ctx); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "top is refcounted"); - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "top is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "top as constant is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "top is NULL"); + TEST_PREDICATE(!_Py_uop_ref_is_not_null(ref), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "top is a constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_ref_is_bottom(ref), "top is bottom"); - sym = make_bottom(ctx); - if (sym == NULL) { + ref = PyJitRef_FromSymbolSteal(make_bottom(ctx)); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "bottom as constant is not NULL"); - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_ref_is_not_null(ref), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "bottom isn't bottom"); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "int is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "int as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "int is NULL"); + TEST_PREDICATE(_Py_uop_ref_is_not_null(ref), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "int is a constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "int as constant is not NULL"); - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + _Py_uop_ref_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + _Py_uop_ref_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); assert(val_42 != NULL); @@ -838,95 +845,86 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(val_43 != NULL); assert(_Py_IsImmortal(val_43)); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "42 isn't refcounted"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "42 as constant isn't 42"); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "42 is not immortal"); - - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "(42 and 42) as constant isn't 42"); - - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); - - sym = _Py_uop_sym_new_type(ctx, &PyBool_Type); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "a bool is not immortal"); - - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + _Py_uop_ref_set_const(ctx, ref, val_42); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "42 is NULL"); + TEST_PREDICATE(_Py_uop_ref_is_not_null(ref), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_ref_is_immortal(ref), "42 is not immortal"); + + _Py_uop_ref_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_ref_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(42 and float) isn't bottom"); + + ref = _Py_uop_ref_new_type(ctx, &PyBool_Type); + TEST_PREDICATE(_Py_uop_ref_is_immortal(ref), "a bool is not immortal"); + + ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + _Py_uop_ref_set_const(ctx, ref, val_42); + _Py_uop_ref_set_const(ctx, ref, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(42 and 43) isn't bottom"); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { - goto fail; - } - TEST_PREDICATE(!_Py_uop_sym_is_skip_refcount(ctx, sym), "type should be refcounted"); - _Py_uop_sym_set_skip_refcount(ctx, sym); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "type should not be refcounted"); - - sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_is_skip_refcount(ctx, sym), "None should not be refcounted"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); - sym = _Py_uop_sym_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); - sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(0) is not False"); - - JitOptSymbol *i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); - JitOptSymbol *i2 = _Py_uop_sym_new_const(ctx, val_43); - JitOptSymbol *array[2] = { i1, i2 }; - sym = _Py_uop_sym_new_tuple(ctx, 2, array); + + ref = _Py_uop_ref_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(None) is not False"); + ref = _Py_uop_ref_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(False) is not False"); + ref = _Py_uop_ref_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(0) is not False"); + + JitOptRef i1 = _Py_uop_ref_new_type(ctx, &PyFloat_Type); + JitOptRef i2 = _Py_uop_ref_new_const(ctx, val_43); + JitOptRef array[2] = { i1, i2 }; + ref = _Py_uop_ref_new_tuple(ctx, 2, array); TEST_PREDICATE( - _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, sym, 0), &PyFloat_Type), + _Py_uop_ref_matches_type(_Py_uop_ref_tuple_getitem(ctx, ref, 0), &PyFloat_Type), "tuple item does not match value used to create tuple" ); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_ref_get_const(ctx, _Py_uop_ref_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; tuple = _PyTuple_FromArray(pair, 2); - sym = _Py_uop_sym_new_const(ctx, tuple); + ref = _Py_uop_ref_new_const(ctx, tuple); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_ref_get_const(ctx, _Py_uop_ref_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); - sym = _Py_uop_sym_new_type(ctx, &PyTuple_Type); + ref = _Py_uop_ref_new_type(ctx, &PyTuple_Type); TEST_PREDICATE( - _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, sym, 42)), + _Py_uop_ref_is_not_null(_Py_uop_ref_tuple_getitem(ctx, ref, 42)), "Unknown tuple item is not narrowed to non-NULL" ); - JitOptSymbol *value = _Py_uop_sym_new_type(ctx, &PyBool_Type); - sym = _Py_uop_sym_new_truthiness(ctx, value, false); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == -1, "truthiness is not unknown"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == false, "truthiness is constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "truthiness is not NULL"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == false, "value is constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == NULL, "value is not NULL"); - _Py_uop_sym_set_const(ctx, sym, Py_False); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "truthiness is not True"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == true, "truthiness is not constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == Py_False, "truthiness is not False"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); + JitOptRef value = _Py_uop_ref_new_type(ctx, &PyBool_Type); + ref = _Py_uop_ref_new_truthiness(ctx, value, false); + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == -1, "truthiness is not unknown"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref) == false, "truthiness is constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "truthiness is not NULL"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, value) == false, "value is constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, value) == NULL, "value is not NULL"); + _Py_uop_ref_set_const(ctx, ref, Py_False); + TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "truthiness is not True"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref) == true, "truthiness is not constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == Py_False, "truthiness is not False"); + TEST_PREDICATE(_Py_uop_ref_is_const(ctx, value) == true, "value is not constant"); + TEST_PREDICATE(_Py_uop_ref_get_const(ctx, value) == Py_True, "value is not True"); _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3070559db8ae57..39ffc1d7e0cbd6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -735,7 +735,7 @@ def visit(stmt: Stmt) -> None: continue #if not tkn.text.startswith(("Py", "_Py", "monitor")): # continue - if tkn.text.startswith(("sym_", "optimize_")): + if tkn.text.startswith(("sym_", "optimize_", "ref_", "PyJitRef")): # Optimize functions continue if tkn.text.endswith("Check"): diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index fda022a44e59cc..dd8420b792f53e 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -72,10 +72,10 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return "JitOptSymbol **" + return "JitOptRef *" if var.type: return var.type - return "JitOptSymbol *" + return "JitOptRef " def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: @@ -124,15 +124,15 @@ def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: local.in_local = True if var.is_array(): if var.size == "1": - out.emit(f"{var.name}[0] = sym_new_not_null(ctx);\n") + out.emit(f"{var.name}[0] = ref_new_not_null(ctx);\n") else: out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") + out.emit(f"{var.name}[_i] = ref_new_not_null(ctx);\n") out.emit("}\n") elif var.name == "null": - out.emit(f"{var.name} = sym_new_null(ctx);\n") + out.emit(f"{var.name} = ref_new_null(ctx);\n") else: - out.emit(f"{var.name} = sym_new_not_null(ctx);\n") + out.emit(f"{var.name} = ref_new_not_null(ctx);\n") class OptimizerEmitter(Emitter): @@ -230,7 +230,7 @@ def generate_abstract_interpreter( declare_variables(override, out, skip_inputs=False) else: declare_variables(uop, out, skip_inputs=True) - stack = Stack(extract_bits=False, cast_type="JitOptSymbol *") + stack = Stack(extract_bits=True) write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) out.start_line() out.emit("break;\n") From a1588358247f7455a8473d215bd47bd037bdcfc5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 23:29:46 +0800 Subject: [PATCH 09/24] refactor more --- Include/internal/pycore_optimizer.h | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 78cea6198d9767..034b93525f9b43 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -183,49 +183,37 @@ typedef enum _JitSymType { typedef struct _jit_opt_known_class { - struct { - uint8_t tag; - }; + uint8_t tag; uint32_t version; PyTypeObject *type; } JitOptKnownClass; typedef struct _jit_opt_known_version { - struct { - uint8_t tag; - }; + uint8_t tag; uint32_t version; } JitOptKnownVersion; typedef struct _jit_opt_known_value { - struct { - uint8_t tag; - }; + uint8_t tag; PyObject *value; } JitOptKnownValue; #define MAX_SYMBOLIC_TUPLE_SIZE 7 typedef struct _jit_opt_tuple { - struct { - uint8_t tag; - }; + uint8_t tag; uint8_t length; uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; } JitOptTuple; typedef struct { - struct { - uint8_t tag; - }; + uint8_t tag; bool invert; uint16_t value; } JitOptTruthiness; typedef union _jit_opt_symbol { - struct { - uint8_t tag; - }; + uint8_t tag; JitOptKnownClass cls; JitOptKnownValue value; JitOptKnownVersion version; From e77f842bcbd8ef3a387a92ec88d22626b156b9a1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 23:31:33 +0800 Subject: [PATCH 10/24] fix debug build --- Python/optimizer_bytecodes.c | 2 +- Python/optimizer_cases.c.h | 2 +- Python/optimizer_symbols.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 333971a2e02828..3b3c08a81e0bbf 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -750,7 +750,7 @@ dummy_func(void) { } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (ref_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f59a9cbd6d217a..d70bf1ee22f55a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1985,7 +1985,7 @@ ctx->done = true; break; } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (ref_is_not_null(self_or_null)) { args--; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 66fa66e21935a3..d0a296a83e1e59 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -745,7 +745,7 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; + ctx->locals_and_stack[i] = PyJitRef_NULL; } #endif From 01004c2404941e3c2dcdde3f11870c2cc175cef2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 28 May 2025 23:33:08 +0800 Subject: [PATCH 11/24] lint --- Python/optimizer_symbols.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index d0a296a83e1e59..994f591c561d6c 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -396,7 +396,7 @@ _Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef ref) } } -JitOptRef +JitOptRef _Py_uop_ref_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); @@ -406,7 +406,7 @@ _Py_uop_ref_new_unknown(JitOptContext *ctx) return PyJitRef_FromSymbolSteal(res); } -JitOptRef +JitOptRef _Py_uop_ref_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); @@ -417,7 +417,7 @@ _Py_uop_ref_new_not_null(JitOptContext *ctx) return PyJitRef_FromSymbolSteal(res); } -JitOptRef +JitOptRef _Py_uop_ref_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); @@ -430,7 +430,7 @@ _Py_uop_ref_new_type(JitOptContext *ctx, PyTypeObject *typ) } // Adds a new reference to const_val, owned by the symbol. -JitOptRef +JitOptRef _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); @@ -443,7 +443,7 @@ _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) return ref; } -JitOptRef +JitOptRef _Py_uop_ref_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); @@ -574,7 +574,7 @@ _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) return -1; } -JitOptRef +JitOptRef _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); @@ -595,7 +595,7 @@ _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) return PyJitRef_FromSymbolSteal(res); } -JitOptRef +JitOptRef _Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); @@ -651,7 +651,7 @@ _Py_uop_ref_is_immortal(JitOptRef ref) return _Py_uop_sym_is_immortal(sym); } -JitOptRef +JitOptRef _Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref); From 0189413bba10f34cc288b59b7c728bd6485b7f1d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 00:50:54 +0800 Subject: [PATCH 12/24] fix upstream --- Python/optimizer_bytecodes.c | 4 ++-- Python/optimizer_cases.c.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d0183a0bec4fe4..1af6aafc0d1297 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -927,10 +927,10 @@ dummy_func(void) { } op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - if (sym_matches_type(iter, &PyTuple_Type)) { + if (ref_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(iter, &PyTuple_Type); + ref_set_type(iter, &PyTuple_Type); } op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index df5a2715121ff3..c8f37aa0b88dae 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1651,12 +1651,12 @@ } case _ITER_CHECK_TUPLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = stack_pointer[-2]; - if (sym_matches_type(iter, &PyTuple_Type)) { + if (ref_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(iter, &PyTuple_Type); + ref_set_type(iter, &PyTuple_Type); break; } From 4a386bf79173f41fb3f90c105e84478237807408 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 00:52:38 +0800 Subject: [PATCH 13/24] reduce diff --- Include/internal/pycore_optimizer.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 034b93525f9b43..72230e7970d0ae 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -181,7 +181,6 @@ typedef enum _JitSymType { JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; - typedef struct _jit_opt_known_class { uint8_t tag; uint32_t version; From ac034a004578ff23f9fdfed11c2cc4c7c57eb19f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 01:14:32 +0800 Subject: [PATCH 14/24] fix for FT --- Include/internal/pycore_optimizer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 72230e7970d0ae..47efc3ce2507d8 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -258,7 +258,7 @@ PyJitRef_Borrow(JitOptRef ref) return (JitOptRef){ .bits = ref.bits | Py_TAG_REFCNT }; } -static const JitOptRef PyJitRef_NULL = { .bits = PyStackRef_NULL_BITS }; +static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = PyStackRef_NULL.bits}; static inline bool PyJitRef_IsNull(JitOptRef ref) From b6e467e79323883ad7904f1c5fdf009e1eb6b1c2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 02:48:07 +0800 Subject: [PATCH 15/24] fix failing tests --- Lib/test/test_generated_cases.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 37046d8e1c02b7..aaed3fba13e758 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -2002,8 +2002,8 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - JitOptSymbol *arg1; - JitOptSymbol *out; + JitOptRef arg1; + JitOptRef out; arg1 = stack_pointer[-1]; out = EGGS(arg1); stack_pointer[-1] = out; @@ -2011,8 +2011,8 @@ def test_overridden_abstract_args(self): } case OP2: { - JitOptSymbol *out; - out = sym_new_not_null(ctx); + JitOptRef out; + out = ref_new_not_null(ctx); stack_pointer[-1] = out; break; } @@ -2036,14 +2036,14 @@ def test_no_overridden_case(self): """ output = """ case OP: { - JitOptSymbol *out; - out = sym_new_not_null(ctx); + JitOptRef out; + out = ref_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - JitOptSymbol *out; + JitOptRef out; out = NULL; stack_pointer[-1] = out; break; @@ -2177,7 +2177,7 @@ def test_validate_uop_unused_output(self): """ output = """ case OP: { - JitOptSymbol *foo; + JitOptRef foo; foo = NULL; stack_pointer[0] = foo; stack_pointer += 1; From 3a3fa9d3138cff29d3ad405b4cecd314989b3a21 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 16:01:50 +0800 Subject: [PATCH 16/24] Fix for disabled GIL --- Include/internal/pycore_optimizer.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 47efc3ce2507d8..d4e65b72212bd8 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -258,7 +258,11 @@ PyJitRef_Borrow(JitOptRef ref) return (JitOptRef){ .bits = ref.bits | Py_TAG_REFCNT }; } -static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = PyStackRef_NULL.bits}; +#ifndef Py_GIL_DISABLED +static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = PyStackRef_NULL_BITS}; +#else +static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = Py_TAG_DEFERRED}; +#endif static inline bool PyJitRef_IsNull(JitOptRef ref) From ab1ad9c62eb575d8dd0080a3daae48bff48dc23b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 16:08:07 +0800 Subject: [PATCH 17/24] fix on FT again --- Include/internal/pycore_optimizer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d4e65b72212bd8..633ffbb3a1f2c0 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -267,7 +267,7 @@ static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = Py_TAG_DEFERRED}; static inline bool PyJitRef_IsNull(JitOptRef ref) { - return ref.bits == PyStackRef_NULL_BITS; + return ref.bits == PyJitRef_NULL.bits; } static inline int From 5d82489bddf113d6f3583a8154cffa9ec46f6e09 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 29 May 2025 16:54:49 +0800 Subject: [PATCH 18/24] Try fix windows --- Include/internal/pycore_optimizer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 633ffbb3a1f2c0..49f71dc0734128 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -259,9 +259,9 @@ PyJitRef_Borrow(JitOptRef ref) } #ifndef Py_GIL_DISABLED -static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = PyStackRef_NULL_BITS}; +static const JitOptRef PyJitRef_NULL = {.bits = PyStackRef_NULL_BITS}; #else -static const JitOptRef PyJitRef_NULL = (JitOptRef){.bits = Py_TAG_DEFERRED}; +static const JitOptRef PyJitRef_NULL = {.bits = Py_TAG_DEFERRED}; #endif static inline bool From 4d9a68e257ecd8c3b7637940fd8dc1f301e1a085 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Jun 2025 22:22:00 +0800 Subject: [PATCH 19/24] Apply code review suggestions from Tomas --- Include/internal/pycore_optimizer.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 49f71dc0734128..3d89fea9ce389a 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -238,18 +238,18 @@ PyJitRef_AsSymbolBorrow(JitOptRef ref) bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); static inline JitOptRef -PyJitRef_FromSymbolSteal(JitOptSymbol *sym) +PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) { - if (sym == NULL || _Py_uop_sym_is_immortal(sym)) { - return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; - } - return (JitOptRef){.bits=(uintptr_t)sym}; + return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; } static inline JitOptRef -PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) +PyJitRef_FromSymbolSteal(JitOptSymbol *sym) { - return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; + if (sym == NULL || _Py_uop_sym_is_immortal(sym)) { + return PyJitRef_FromSymbolBorrow(sym); + } + return (JitOptRef){.bits=(uintptr_t)sym}; } static inline JitOptRef From 2bbd47a9514d701eff96637720bffa1e4dfd16f7 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Jun 2025 23:07:08 +0800 Subject: [PATCH 20/24] call the functions sym instead of ref --- Include/internal/pycore_optimizer.h | 52 +- Lib/test/test_generated_cases.py | 4 +- Python/optimizer_analysis.c | 60 +- Python/optimizer_bytecodes.c | 548 ++++++++-------- Python/optimizer_cases.c.h | 642 +++++++++---------- Python/optimizer_symbols.c | 258 ++++---- Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/optimizer_generator.py | 8 +- 8 files changed, 787 insertions(+), 787 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 3d89fea9ce389a..88acea076dbc80 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -235,7 +235,7 @@ PyJitRef_AsSymbolBorrow(JitOptRef ref) return JIT_BITS_TO_PTR_MASKED(ref); } -bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); +bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym); static inline JitOptRef PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) @@ -246,7 +246,7 @@ PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) static inline JitOptRef PyJitRef_FromSymbolSteal(JitOptSymbol *sym) { - if (sym == NULL || _Py_uop_sym_is_immortal(sym)) { + if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) { return PyJitRef_FromSymbolBorrow(sym); } return (JitOptRef){.bits=(uintptr_t)sym}; @@ -311,31 +311,31 @@ typedef struct _JitOptContext { JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; } JitOptContext; -extern bool _Py_uop_ref_is_null(JitOptRef sym); -extern bool _Py_uop_ref_is_not_null(JitOptRef sym); -extern bool _Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef sym); -extern PyObject *_Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef sym); -extern JitOptRef _Py_uop_ref_new_unknown(JitOptContext *ctx); -extern JitOptRef _Py_uop_ref_new_not_null(JitOptContext *ctx); -extern JitOptRef _Py_uop_ref_new_type( +extern bool _Py_uop_sym_is_null(JitOptRef sym); +extern bool _Py_uop_sym_is_not_null(JitOptRef sym); +extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef sym); +extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_unknown(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_not_null(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_type( JitOptContext *ctx, PyTypeObject *typ); -extern JitOptRef _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val); -extern JitOptRef _Py_uop_ref_new_null(JitOptContext *ctx); -extern bool _Py_uop_ref_has_type(JitOptRef sym); -extern bool _Py_uop_ref_matches_type(JitOptRef sym, PyTypeObject *typ); -extern bool _Py_uop_ref_matches_type_version(JitOptRef sym, unsigned int version); -extern void _Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef sym); -extern void _Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef sym); -extern void _Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); -extern bool _Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef sym, unsigned int version); -extern void _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef sym, PyObject *const_val); -extern bool _Py_uop_ref_is_bottom(JitOptRef sym); -extern int _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef sym); -extern PyTypeObject *_Py_uop_ref_get_type(JitOptRef sym); -extern JitOptRef _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); -extern JitOptRef _Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item); -extern int _Py_uop_ref_tuple_length(JitOptRef sym); -extern JitOptRef _Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); +extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); +extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); +extern bool _Py_uop_sym_has_type(JitOptRef sym); +extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version); +extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef sym, unsigned int version); +extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(JitOptRef sym); +extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef sym); +extern PyTypeObject *_Py_uop_sym_get_type(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); +extern JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item); +extern int _Py_uop_sym_tuple_length(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index aaed3fba13e758..b609c2bb297078 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -2012,7 +2012,7 @@ def test_overridden_abstract_args(self): case OP2: { JitOptRef out; - out = ref_new_not_null(ctx); + out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } @@ -2037,7 +2037,7 @@ def test_no_overridden_case(self): output = """ case OP: { JitOptRef out; - out = ref_new_not_null(ctx); + out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 874d4482f24f5f..de337f637163ba 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -315,33 +315,33 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, INST->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ -#define ref_is_not_null _Py_uop_ref_is_not_null -#define ref_is_const _Py_uop_ref_is_const -#define ref_get_const _Py_uop_ref_get_const -#define ref_new_unknown _Py_uop_ref_new_unknown -#define ref_new_not_null _Py_uop_ref_new_not_null -#define ref_new_type _Py_uop_ref_new_type -#define ref_is_null _Py_uop_ref_is_null -#define ref_new_const _Py_uop_ref_new_const -#define ref_new_null _Py_uop_ref_new_null -#define ref_has_type _Py_uop_ref_has_type -#define ref_get_type _Py_uop_ref_get_type -#define ref_matches_type _Py_uop_ref_matches_type -#define ref_matches_type_version _Py_uop_ref_matches_type_version -#define ref_set_null(SYM) _Py_uop_ref_set_null(ctx, SYM) -#define ref_set_non_null(SYM) _Py_uop_ref_set_non_null(ctx, SYM) -#define ref_set_type(SYM, TYPE) _Py_uop_ref_set_type(ctx, SYM, TYPE) -#define ref_set_type_version(SYM, VERSION) _Py_uop_ref_set_type_version(ctx, SYM, VERSION) -#define ref_set_const(SYM, CNST) _Py_uop_ref_set_const(ctx, SYM, CNST) -#define ref_is_bottom _Py_uop_ref_is_bottom -#define ref_truthiness _Py_uop_ref_truthiness +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_has_type _Py_uop_sym_has_type +#define sym_get_type _Py_uop_sym_get_type +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_sym_is_bottom +#define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop -#define ref_new_tuple _Py_uop_ref_new_tuple -#define ref_tuple_getitem _Py_uop_ref_tuple_getitem -#define ref_tuple_length _Py_uop_ref_tuple_length -#define ref_is_immortal _Py_uop_ref_is_immortal -#define ref_new_truthiness _Py_uop_ref_new_truthiness +#define sym_new_tuple _Py_uop_sym_new_tuple +#define sym_tuple_getitem _Py_uop_sym_tuple_getitem +#define sym_tuple_length _Py_uop_sym_tuple_length +#define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_new_truthiness _Py_uop_sym_new_truthiness static int optimize_to_bool( @@ -350,16 +350,16 @@ optimize_to_bool( JitOptRef value, JitOptRef *result_ptr) { - if (ref_matches_type(value, &PyBool_Type)) { + if (sym_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); *result_ptr = value; return 1; } - int truthiness = ref_truthiness(ctx, value); + int truthiness = sym_truthiness(ctx, value); if (truthiness >= 0) { PyObject *load = truthiness ? Py_True : Py_False; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - *result_ptr = ref_new_const(ctx, load); + *result_ptr = sym_new_const(ctx, load); return 1; } return 0; @@ -386,10 +386,10 @@ lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup); - return ref_new_const(ctx, lookup); + return sym_new_const(ctx, lookup); } } - return ref_new_not_null(ctx); + return sym_new_not_null(ctx); } /* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 1af6aafc0d1297..7d65a9187b2497 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -9,32 +9,32 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; /* Shortened forms for convenience */ -#define ref_is_not_null _Py_uop_ref_is_not_null -#define ref_is_const _Py_uop_ref_is_const -#define ref_get_const _Py_uop_ref_get_const -#define ref_new_unknown _Py_uop_ref_new_unknown -#define ref_new_not_null _Py_uop_ref_new_not_null -#define ref_new_type _Py_uop_ref_new_type -#define ref_is_null _Py_uop_ref_is_null -#define ref_new_const _Py_uop_ref_new_const -#define ref_new_null _Py_uop_ref_new_null -#define ref_matches_type _Py_uop_ref_matches_type -#define ref_matches_type_version _Py_uop_ref_matches_type_version -#define ref_get_type _Py_uop_ref_get_type -#define ref_has_type _Py_uop_ref_has_type -#define ref_set_null(SYM) _Py_uop_ref_set_null(ctx, SYM) -#define ref_set_non_null(SYM) _Py_uop_ref_set_non_null(ctx, SYM) -#define ref_set_type(SYM, TYPE) _Py_uop_ref_set_type(ctx, SYM, TYPE) -#define ref_set_type_version(SYM, VERSION) _Py_uop_ref_set_type_version(ctx, SYM, VERSION) -#define ref_set_const(SYM, CNST) _Py_uop_ref_set_const(ctx, SYM, CNST) -#define ref_is_bottom _Py_uop_ref_is_bottom +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_get_type _Py_uop_sym_get_type +#define sym_has_type _Py_uop_sym_has_type +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop -#define ref_new_tuple _Py_uop_ref_new_tuple -#define ref_tuple_getitem _Py_uop_ref_tuple_getitem -#define ref_tuple_length _Py_uop_ref_tuple_length -#define ref_is_immortal _Py_uop_ref_is_immortal -#define ref_new_truthiness _Py_uop_ref_new_truthiness +#define sym_new_tuple _Py_uop_sym_new_tuple +#define sym_tuple_getitem _Py_uop_sym_tuple_getitem +#define sym_tuple_length _Py_uop_sym_tuple_length +#define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_new_truthiness _Py_uop_sym_new_truthiness extern int optimize_to_bool( @@ -77,7 +77,7 @@ dummy_func(void) { op(_LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. - if (ref_is_null(value)) { + if (sym_is_null(value)) { ctx->done = true; } } @@ -92,7 +92,7 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - JitOptRef temp = ref_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -101,38 +101,38 @@ dummy_func(void) { } op(_PUSH_NULL, (-- res)) { - res = ref_new_null(ctx); + res = sym_new_null(ctx); } op(_GUARD_TOS_INT, (value -- value)) { - if (ref_matches_type(value, &PyLong_Type)) { + if (sym_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyLong_Type); + sym_set_type(value, &PyLong_Type); } op(_GUARD_NOS_INT, (left, unused -- left, unused)) { - if (ref_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(left, &PyLong_Type); + sym_set_type(left, &PyLong_Type); } op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { - if (type == ref_get_const(ctx, owner)) { + if (type == sym_get_const(ctx, owner)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { - ref_set_const(owner, type); + sym_set_const(owner, type); } } } op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { assert(type_version); - if (ref_matches_type_version(owner, type_version)) { + if (sym_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { // add watcher so that whenever the type changes we invalidate this @@ -144,7 +144,7 @@ dummy_func(void) { // if it wasn't this means that the type version was previously set to something else // and we set the owner to bottom, so we don't need to add a watcher because we must have // already added one earlier. - if (ref_set_type_version(owner, type_version)) { + if (sym_set_type_version(owner, type_version)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); } @@ -154,27 +154,27 @@ dummy_func(void) { } op(_GUARD_TOS_FLOAT, (value -- value)) { - if (ref_matches_type(value, &PyFloat_Type)) { + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { - if (ref_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(left, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); } op(_BINARY_OP, (lhs, rhs -- res)) { - bool lhs_int = ref_matches_type(lhs, &PyLong_Type); - bool rhs_int = ref_matches_type(rhs, &PyLong_Type); - bool lhs_float = ref_matches_type(lhs, &PyFloat_Type); - bool rhs_float = ref_matches_type(rhs, &PyFloat_Type); + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { // There's something other than an int or float involved: - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { // This one's fun... the *type* of the result depends on the @@ -191,110 +191,110 @@ dummy_func(void) { if (rhs_float) { // Case D, E, F, or G... can't know without the sign of the LHS // or whether the RHS is whole, which isn't worth the effort: - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } else if (lhs_float) { // Case C: - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } - else if (!ref_is_const(ctx, rhs)) { + else if (!sym_is_const(ctx, rhs)) { // Case A or B... can't know without the sign of the RHS: - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)ref_get_const(ctx, rhs))) { + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { // Case B: - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } else { // Case A: - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } else if (lhs_int && rhs_int) { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -303,22 +303,22 @@ dummy_func(void) { } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -327,22 +327,22 @@ dummy_func(void) { } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } // TODO (gh-134584): Move this to the optimizer generator. if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -351,35 +351,35 @@ dummy_func(void) { } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); - assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { JitOptRef res; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); - assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } // _STORE_FAST: GETLOCAL(this_instr->operand0) = res; @@ -391,109 +391,109 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) { - assert(ref_matches_type(tuple_st, &PyTuple_Type)); - if (ref_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(ref_get_const(ctx, sub_st))); - long index = PyLong_AsLong(ref_get_const(ctx, sub_st)); + assert(sym_matches_type(tuple_st, &PyTuple_Type)); + if (sym_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); + long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = ref_tuple_length(tuple_st); + int tuple_length = sym_tuple_length(tuple_st); if (tuple_length == -1) { // Unknown length - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } else { assert(index < tuple_length); - res = ref_tuple_getitem(ctx, tuple_st, index); + res = sym_tuple_getitem(ctx, tuple_st, index); } } else { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } } op(_TO_BOOL, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_truthiness(ctx, value, true); + res = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_BOOL, (value -- value)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { - ref_set_type(value, &PyBool_Type); - value = ref_new_truthiness(ctx, value, true); + sym_set_type(value, &PyBool_Type); + value = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_INT, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - ref_set_type(value, &PyLong_Type); - res = ref_new_truthiness(ctx, value, true); + sym_set_type(value, &PyLong_Type); + res = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_LIST, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_NONE, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - ref_set_const(value, Py_None); - res = ref_new_const(ctx, Py_False); + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); } } op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { - if (ref_matches_type(nos, &PyUnicode_Type)) { + if (sym_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyUnicode_Type); + sym_set_type(nos, &PyUnicode_Type); } op(_GUARD_TOS_UNICODE, (value -- value)) { - if (ref_matches_type(value, &PyUnicode_Type)) { + if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyUnicode_Type); + sym_set_type(value, &PyUnicode_Type); } op(_TO_BOOL_STR, (value -- res)) { int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_truthiness(ctx, value, true); + res = sym_new_truthiness(ctx, value, true); } } op(_UNARY_NOT, (value -- res)) { - ref_set_type(value, &PyBool_Type); - res = ref_new_truthiness(ctx, value, false); + sym_set_type(value, &PyBool_Type); + res = sym_new_truthiness(ctx, value, false); } op(_COMPARE_OP, (left, right -- res)) { if (oparg & 16) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } else { - res = _Py_uop_ref_new_not_null(ctx); + res = _Py_uop_sym_new_not_null(ctx); } } op(_COMPARE_OP_INT, (left, right -- res)) { - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(ref_get_const(ctx, left), - ref_get_const(ctx, right), + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), + sym_get_const(ctx, right), oparg >> 5); if (tmp == NULL) { goto error; @@ -501,42 +501,42 @@ dummy_func(void) { assert(PyBool_Check(tmp)); assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = ref_new_const(ctx, tmp); + res = sym_new_const(ctx, tmp); Py_DECREF(tmp); } else { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } } op(_COMPARE_OP_FLOAT, (left, right -- res)) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_STR, (left, right -- res)) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } op(_IS_OP, (left, right -- b)) { - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP, (left, right -- b)) { - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP_SET, (left, right -- b)) { - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); } op(_CONTAINS_OP_DICT, (left, right -- b)) { - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); } op(_LOAD_CONST, (-- value)) { PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(ref_new_const(ctx, val)); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { @@ -544,35 +544,35 @@ dummy_func(void) { assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(ref_new_const(ctx, val)); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -588,7 +588,7 @@ dummy_func(void) { } op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)offset; } @@ -596,8 +596,8 @@ dummy_func(void) { (void)dict_version; (void)index; attr = PyJitRef_NULL; - if (ref_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)ref_get_const(ctx, owner); + if (sym_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; uint64_t watched_mutations = get_mutations(dict); @@ -605,20 +605,20 @@ dummy_func(void) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = ref_new_const(ctx, res); + attr = sym_new_const(ctx, res); } } } if (PyJitRef_IsNull(attr)) { /* No conversion made. We don't know what `attr` is. */ - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); } } op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); - null[0] = ref_new_null(ctx); + null[0] = sym_new_null(ctx); } else { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -627,25 +627,25 @@ dummy_func(void) { op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { (void)owner; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); if (oparg & 1) { - self_or_null[0] = ref_new_unknown(ctx); + self_or_null[0] = sym_new_unknown(ctx); } } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr)) { - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)hint; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)index; } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = (PyTypeObject *)ref_get_const(ctx, owner); + PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -654,7 +654,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -663,7 +663,7 @@ dummy_func(void) { op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -672,7 +672,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -682,7 +682,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -692,7 +692,7 @@ dummy_func(void) { op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -707,26 +707,26 @@ dummy_func(void) { } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - if (ref_is_const(ctx, callable) && ref_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(ref_get_const(ctx, callable))); + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)ref_get_const(ctx, callable); + this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); } - ref_set_type(callable, &PyFunction_Type); + sym_set_type(callable, &PyFunction_Type); } op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - assert(ref_matches_type(callable, &PyFunction_Type)); - if (ref_is_const(ctx, callable)) { - if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)ref_get_const(ctx, callable); + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(ctx, callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !ref_is_null(self_or_null)) { + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } } @@ -734,8 +734,8 @@ dummy_func(void) { } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - ref_set_null(null); - ref_set_type(callable, &PyMethod_Type); + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -752,13 +752,13 @@ dummy_func(void) { assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); - if (ref_is_not_null(self_or_null)) { + if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM args--; argcount++; } - if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { new_frame = frame_new(ctx, co, 0, args, argcount); } else { new_frame = frame_new(ctx, co, 0, NULL, 0); @@ -768,8 +768,8 @@ dummy_func(void) { op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)args; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { @@ -792,8 +792,8 @@ dummy_func(void) { op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)type_version; (void)args; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { @@ -831,7 +831,7 @@ dummy_func(void) { ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); /* Stack space handling */ assert(corresponding_check_stack == NULL); @@ -849,7 +849,7 @@ dummy_func(void) { } op(_YIELD_VALUE, (unused -- value)) { - value = ref_new_unknown(ctx); + value = sym_new_unknown(ctx); } op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) { @@ -913,7 +913,7 @@ dummy_func(void) { (void)top; /* This has to be done manually */ for (int i = 0; i < oparg; i++) { - values[i] = ref_new_unknown(ctx); + values[i] = sym_new_unknown(ctx); } } @@ -922,47 +922,47 @@ dummy_func(void) { /* This has to be done manually */ int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = ref_new_unknown(ctx); + values[i] = sym_new_unknown(ctx); } } op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { - if (ref_matches_type(iter, &PyTuple_Type)) { + if (sym_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(iter, &PyTuple_Type); + sym_set_type(iter, &PyTuple_Type); } op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { - next = ref_new_type(ctx, &PyLong_Type); + next = sym_new_type(ctx, &PyLong_Type); } op(_CALL_TYPE_1, (unused, unused, arg -- res)) { - if (ref_has_type(arg)) { - res = ref_new_const(ctx, (PyObject *)ref_get_type(arg)); + if (sym_has_type(arg)) { + res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); } else { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } } op(_CALL_STR_1, (unused, unused, arg -- res)) { - if (ref_matches_type(arg, &PyUnicode_Type)) { + if (sym_matches_type(arg, &PyUnicode_Type)) { // e.g. str('foo') or str(foo) where foo is known to be a string res = arg; } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } } op(_CALL_ISINSTANCE, (unused, unused, instance, cls -- res)) { // the result is always a bool, but sometimes we can // narrow it down to True or False - res = ref_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = ref_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)ref_get_const(ctx, cls); - if (inst_type && cls_o && ref_matches_type(cls, &PyType_Type)) { + res = sym_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = sym_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); + if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { // isinstance(inst, cls) where both inst and cls have // known types, meaning we can deduce either True or False @@ -971,50 +971,50 @@ dummy_func(void) { if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { out = Py_True; } - ref_set_const(res, out); + sym_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } } op(_GUARD_IS_TRUE_POP, (flag -- )) { - if (ref_is_const(ctx, flag)) { - PyObject *value = ref_get_const(ctx, flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } - ref_set_const(flag, Py_True); + sym_set_const(flag, Py_True); } op(_GUARD_IS_FALSE_POP, (flag -- )) { - if (ref_is_const(ctx, flag)) { - PyObject *value = ref_get_const(ctx, flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } - ref_set_const(flag, Py_False); + sym_set_const(flag, Py_False); } op(_GUARD_IS_NONE_POP, (val -- )) { - if (ref_is_const(ctx, val)) { - PyObject *value = ref_get_const(ctx, val); + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (ref_has_type(val)) { - assert(!ref_matches_type(val, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - ref_set_const(val, Py_None); + sym_set_const(val, Py_None); } op(_GUARD_IS_NOT_NONE_POP, (val -- )) { - if (ref_is_const(ctx, val)) { - PyObject *value = ref_get_const(ctx, val); + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (ref_has_type(val)) { - assert(!ref_matches_type(val, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } } @@ -1028,13 +1028,13 @@ dummy_func(void) { } op(_INSERT_NULL, (self -- method_and_self[2])) { - method_and_self[0] = ref_new_null(ctx); + method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; } op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) { - method_and_self[0] = ref_new_not_null(ctx); - method_and_self[1] = ref_new_unknown(ctx); + method_and_self[0] = sym_new_not_null(ctx); + method_and_self[1] = sym_new_unknown(ctx); } op(_JUMP_TO_TOP, (--)) { @@ -1048,154 +1048,154 @@ dummy_func(void) { op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - res = ref_new_const(ctx, Py_True); + res = sym_new_const(ctx, Py_True); } op(_BUILD_TUPLE, (values[oparg] -- tup)) { - tup = ref_new_tuple(ctx, oparg, values); + tup = sym_new_tuple(ctx, oparg, values); } op(_BUILD_LIST, (values[oparg] -- list)) { - list = ref_new_type(ctx, &PyList_Type); + list = sym_new_type(ctx, &PyList_Type); } op(_BUILD_SLICE, (args[oparg] -- slice)) { - slice = ref_new_type(ctx, &PySlice_Type); + slice = sym_new_type(ctx, &PySlice_Type); } op(_BUILD_MAP, (values[oparg*2] -- map)) { - map = ref_new_type(ctx, &PyDict_Type); + map = sym_new_type(ctx, &PyDict_Type); } op(_BUILD_STRING, (pieces[oparg] -- str)) { - str = ref_new_type(ctx, &PyUnicode_Type); + str = sym_new_type(ctx, &PyUnicode_Type); } op(_BUILD_SET, (values[oparg] -- set)) { - set = ref_new_type(ctx, &PySet_Type); + set = sym_new_type(ctx, &PySet_Type); } op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { - val0 = ref_tuple_getitem(ctx, seq, 0); - val1 = ref_tuple_getitem(ctx, seq, 1); + val0 = sym_tuple_getitem(ctx, seq, 0); + val1 = sym_tuple_getitem(ctx, seq, 1); } op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { for (int i = 0; i < oparg; i++) { - values[i] = ref_tuple_getitem(ctx, seq, oparg - i - 1); + values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } } op(_CALL_TUPLE_1, (callable, null, arg -- res)) { - if (ref_matches_type(arg, &PyTuple_Type)) { + if (sym_matches_type(arg, &PyTuple_Type)) { // e.g. tuple((1, 2)) or tuple(foo) where foo is known to be a tuple res = arg; } else { - res = ref_new_type(ctx, &PyTuple_Type); + res = sym_new_type(ctx, &PyTuple_Type); } } op(_GUARD_TOS_LIST, (tos -- tos)) { - if (ref_matches_type(tos, &PyList_Type)) { + if (sym_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyList_Type); + sym_set_type(tos, &PyList_Type); } op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { - if (ref_matches_type(nos, &PyList_Type)) { + if (sym_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyList_Type); + sym_set_type(nos, &PyList_Type); } op(_GUARD_TOS_TUPLE, (tos -- tos)) { - if (ref_matches_type(tos, &PyTuple_Type)) { + if (sym_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyTuple_Type); + sym_set_type(tos, &PyTuple_Type); } op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { - if (ref_matches_type(nos, &PyTuple_Type)) { + if (sym_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyTuple_Type); + sym_set_type(nos, &PyTuple_Type); } op(_GUARD_TOS_DICT, (tos -- tos)) { - if (ref_matches_type(tos, &PyDict_Type)) { + if (sym_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyDict_Type); + sym_set_type(tos, &PyDict_Type); } op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { - if (ref_matches_type(nos, &PyDict_Type)) { + if (sym_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyDict_Type); + sym_set_type(nos, &PyDict_Type); } op(_GUARD_TOS_ANY_SET, (tos -- tos)) { - if (ref_matches_type(tos, &PySet_Type) || - ref_matches_type(tos, &PyFrozenSet_Type)) + if (sym_matches_type(tos, &PySet_Type) || + sym_matches_type(tos, &PyFrozenSet_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } } op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { - if (ref_is_null(null)) { + if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_null(null); + sym_set_null(null); } op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { - if (ref_is_not_null(nos)) { + if (sym_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_non_null(nos); + sym_set_non_null(nos); } op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { - if (ref_is_null(null)) { + if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_null(null); + sym_set_null(null); } op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { - if (ref_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyType_Type); + sym_set_const(callable, (PyObject *)&PyType_Type); } op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { - if (ref_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyTuple_Type); + sym_set_const(callable, (PyObject *)&PyTuple_Type); } op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { - if (ref_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyUnicode_Type); + sym_set_const(callable, (PyObject *)&PyUnicode_Type); } op(_CALL_LEN, (unused, unused, unused -- res)) { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } op(_GET_LEN, (obj -- obj, len)) { - int tuple_length = ref_tuple_length(obj); + int tuple_length = sym_tuple_length(obj); if (tuple_length == -1) { - len = ref_new_type(ctx, &PyLong_Type); + len = sym_new_type(ctx, &PyLong_Type); } else { assert(tuple_length >= 0); @@ -1206,33 +1206,33 @@ dummy_func(void) { if (_Py_IsImmortal(temp)) { REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - len = ref_new_const(ctx, temp); + len = sym_new_const(ctx, temp); Py_DECREF(temp); } } op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (ref_get_const(ctx, callable) == len) { + if (sym_get_const(ctx, callable) == len) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, len); + sym_set_const(callable, len); } op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (ref_get_const(ctx, callable) == isinstance) { + if (sym_get_const(ctx, callable) == isinstance) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, isinstance); + sym_set_const(callable, isinstance); } op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (ref_get_const(ctx, callable) == list_append) { + if (sym_get_const(ctx, callable) == list_append) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, list_append); + sym_set_const(callable, list_append); } // END BYTECODES // diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c8f37aa0b88dae..68caa24947755e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -28,7 +28,7 @@ case _LOAD_FAST_CHECK: { JitOptRef value; value = GETLOCAL(oparg); - if (ref_is_null(value)) { + if (sym_is_null(value)) { ctx->done = true; } stack_pointer[0] = value; @@ -58,7 +58,7 @@ case _LOAD_FAST_AND_CLEAR: { JitOptRef value; value = GETLOCAL(oparg); - JitOptRef temp = ref_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -70,7 +70,7 @@ JitOptRef value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(ref_new_const(ctx, val)); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -83,7 +83,7 @@ assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(ref_new_const(ctx, val)); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -113,7 +113,7 @@ case _PUSH_NULL: { JitOptRef res; - res = ref_new_null(ctx); + res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -134,7 +134,7 @@ case _END_SEND: { JitOptRef val; - val = ref_new_not_null(ctx); + val = sym_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -143,7 +143,7 @@ case _UNARY_NEGATIVE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } @@ -152,8 +152,8 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - ref_set_type(value, &PyBool_Type); - res = ref_new_truthiness(ctx, value, false); + sym_set_type(value, &PyBool_Type); + res = sym_new_truthiness(ctx, value, false); stack_pointer[-1] = res; break; } @@ -164,7 +164,7 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_truthiness(ctx, value, true); + res = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; @@ -175,8 +175,8 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { - ref_set_type(value, &PyBool_Type); - value = ref_new_truthiness(ctx, value, true); + sym_set_type(value, &PyBool_Type); + value = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = value; break; @@ -188,8 +188,8 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - ref_set_type(value, &PyLong_Type); - res = ref_new_truthiness(ctx, value, true); + sym_set_type(value, &PyLong_Type); + res = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; @@ -198,20 +198,20 @@ case _GUARD_NOS_LIST: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_matches_type(nos, &PyList_Type)) { + if (sym_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyList_Type); + sym_set_type(nos, &PyList_Type); break; } case _GUARD_TOS_LIST: { JitOptRef tos; tos = stack_pointer[-1]; - if (ref_matches_type(tos, &PyList_Type)) { + if (sym_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyList_Type); + sym_set_type(tos, &PyList_Type); break; } @@ -225,7 +225,7 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } stack_pointer[-1] = res; break; @@ -237,8 +237,8 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - ref_set_const(value, Py_None); - res = ref_new_const(ctx, Py_False); + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); } stack_pointer[-1] = res; break; @@ -247,20 +247,20 @@ case _GUARD_NOS_UNICODE: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_matches_type(nos, &PyUnicode_Type)) { + if (sym_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyUnicode_Type); + sym_set_type(nos, &PyUnicode_Type); break; } case _GUARD_TOS_UNICODE: { JitOptRef value; value = stack_pointer[-1]; - if (ref_matches_type(value, &PyUnicode_Type)) { + if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyUnicode_Type); + sym_set_type(value, &PyUnicode_Type); break; } @@ -270,7 +270,7 @@ value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { - res = ref_new_truthiness(ctx, value, true); + res = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; @@ -279,14 +279,14 @@ case _REPLACE_WITH_TRUE: { JitOptRef res; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); - res = ref_new_const(ctx, Py_True); + res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } @@ -294,20 +294,20 @@ case _GUARD_NOS_INT: { JitOptRef left; left = stack_pointer[-2]; - if (ref_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(left, &PyLong_Type); + sym_set_type(left, &PyLong_Type); break; } case _GUARD_TOS_INT: { JitOptRef value; value = stack_pointer[-1]; - if (ref_matches_type(value, &PyLong_Type)) { + if (sym_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyLong_Type); + sym_set_type(value, &PyLong_Type); break; } @@ -317,22 +317,22 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -345,22 +345,22 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -373,22 +373,22 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)ref_get_const(ctx, left), - (PyLongObject *)ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -398,20 +398,20 @@ case _GUARD_NOS_FLOAT: { JitOptRef left; left = stack_pointer[-2]; - if (ref_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(left, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); break; } case _GUARD_TOS_FLOAT: { JitOptRef value; value = stack_pointer[-1]; - if (ref_matches_type(value, &PyFloat_Type)) { + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(value, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); break; } @@ -421,23 +421,23 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -453,23 +453,23 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -485,23 +485,23 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyFloat_CheckExact(ref_get_const(ctx, left))); - assert(PyFloat_CheckExact(ref_get_const(ctx, right))); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(ref_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(ref_get_const(ctx, right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { @@ -513,7 +513,7 @@ case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -522,7 +522,7 @@ case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -531,7 +531,7 @@ case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -544,21 +544,21 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); - assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -571,18 +571,18 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; JitOptRef res; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(ref_get_const(ctx, left))); - assert(PyUnicode_CheckExact(ref_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(ref_get_const(ctx, left), ref_get_const(ctx, right)); + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); + assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } - res = ref_new_const(ctx, temp); + res = sym_new_const(ctx, temp); Py_DECREF(temp); } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } GETLOCAL(this_instr->operand0) = res; stack_pointer += -2; @@ -596,7 +596,7 @@ case _BINARY_OP_EXTEND: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -605,7 +605,7 @@ case _BINARY_SLICE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -620,7 +620,7 @@ case _BINARY_OP_SUBSCR_LIST_INT: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -629,7 +629,7 @@ case _BINARY_OP_SUBSCR_LIST_SLICE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -638,7 +638,7 @@ case _BINARY_OP_SUBSCR_STR_INT: { JitOptRef res; - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -648,20 +648,20 @@ case _GUARD_NOS_TUPLE: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_matches_type(nos, &PyTuple_Type)) { + if (sym_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyTuple_Type); + sym_set_type(nos, &PyTuple_Type); break; } case _GUARD_TOS_TUPLE: { JitOptRef tos; tos = stack_pointer[-1]; - if (ref_matches_type(tos, &PyTuple_Type)) { + if (sym_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyTuple_Type); + sym_set_type(tos, &PyTuple_Type); break; } @@ -671,22 +671,22 @@ JitOptRef res; sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; - assert(ref_matches_type(tuple_st, &PyTuple_Type)); - if (ref_is_const(ctx, sub_st)) { - assert(PyLong_CheckExact(ref_get_const(ctx, sub_st))); - long index = PyLong_AsLong(ref_get_const(ctx, sub_st)); + assert(sym_matches_type(tuple_st, &PyTuple_Type)); + if (sym_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); + long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = ref_tuple_length(tuple_st); + int tuple_length = sym_tuple_length(tuple_st); if (tuple_length == -1) { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } else { assert(index < tuple_length); - res = ref_tuple_getitem(ctx, tuple_st, index); + res = sym_tuple_getitem(ctx, tuple_st, index); } } else { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -697,26 +697,26 @@ case _GUARD_NOS_DICT: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_matches_type(nos, &PyDict_Type)) { + if (sym_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(nos, &PyDict_Type); + sym_set_type(nos, &PyDict_Type); break; } case _GUARD_TOS_DICT: { JitOptRef tos; tos = stack_pointer[-1]; - if (ref_matches_type(tos, &PyDict_Type)) { + if (sym_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(tos, &PyDict_Type); + sym_set_type(tos, &PyDict_Type); break; } case _BINARY_OP_SUBSCR_DICT: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -725,7 +725,7 @@ case _BINARY_OP_SUBSCR_CHECK_FUNC: { JitOptRef getitem; - getitem = ref_new_not_null(ctx); + getitem = sym_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -780,14 +780,14 @@ case _CALL_INTRINSIC_1: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -823,14 +823,14 @@ case _GET_AITER: { JitOptRef iter; - iter = ref_new_not_null(ctx); + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { JitOptRef awaitable; - awaitable = ref_new_not_null(ctx); + awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -839,7 +839,7 @@ case _GET_AWAITABLE: { JitOptRef iter; - iter = ref_new_not_null(ctx); + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -856,7 +856,7 @@ case _YIELD_VALUE: { JitOptRef value; - value = ref_new_unknown(ctx); + value = sym_new_unknown(ctx); stack_pointer[-1] = value; break; } @@ -869,7 +869,7 @@ case _LOAD_COMMON_CONSTANT: { JitOptRef value; - value = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -878,7 +878,7 @@ case _LOAD_BUILD_CLASS: { JitOptRef bc; - bc = ref_new_not_null(ctx); + bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -902,7 +902,7 @@ top = &stack_pointer[-1 + oparg]; (void)top; for (int i = 0; i < oparg; i++) { - values[i] = ref_new_unknown(ctx); + values[i] = sym_new_unknown(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -914,8 +914,8 @@ JitOptRef val1; JitOptRef val0; seq = stack_pointer[-1]; - val0 = ref_tuple_getitem(ctx, seq, 0); - val1 = ref_tuple_getitem(ctx, seq, 1); + val0 = sym_tuple_getitem(ctx, seq, 0); + val1 = sym_tuple_getitem(ctx, seq, 1); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -929,7 +929,7 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { - values[i] = ref_tuple_getitem(ctx, seq, oparg - i - 1); + values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -940,7 +940,7 @@ JitOptRef *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = ref_new_not_null(ctx); + values[_i] = sym_new_not_null(ctx); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -955,7 +955,7 @@ (void)top; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = ref_new_unknown(ctx); + values[i] = sym_new_unknown(ctx); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -986,7 +986,7 @@ case _LOAD_LOCALS: { JitOptRef locals; - locals = ref_new_not_null(ctx); + locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -997,7 +997,7 @@ case _LOAD_NAME: { JitOptRef v; - v = ref_new_not_null(ctx); + v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1007,7 +1007,7 @@ case _LOAD_GLOBAL: { JitOptRef *res; res = &stack_pointer[0]; - res[0] = ref_new_not_null(ctx); + res[0] = sym_new_not_null(ctx); stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1018,7 +1018,7 @@ null = &stack_pointer[0]; if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); - null[0] = ref_new_null(ctx); + null[0] = sym_new_null(ctx); } else { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1034,7 +1034,7 @@ case _LOAD_GLOBAL_MODULE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1043,7 +1043,7 @@ case _LOAD_GLOBAL_BUILTINS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1064,14 +1064,14 @@ case _LOAD_FROM_DICT_OR_DEREF: { JitOptRef value; - value = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { JitOptRef value; - value = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1090,7 +1090,7 @@ case _BUILD_STRING: { JitOptRef str; - str = ref_new_type(ctx, &PyUnicode_Type); + str = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1099,7 +1099,7 @@ case _BUILD_INTERPOLATION: { JitOptRef interpolation; - interpolation = ref_new_not_null(ctx); + interpolation = sym_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -1108,7 +1108,7 @@ case _BUILD_TEMPLATE: { JitOptRef template; - template = ref_new_not_null(ctx); + template = sym_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1119,7 +1119,7 @@ JitOptRef *values; JitOptRef tup; values = &stack_pointer[-oparg]; - tup = ref_new_tuple(ctx, oparg, values); + tup = sym_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1128,7 +1128,7 @@ case _BUILD_LIST: { JitOptRef list; - list = ref_new_type(ctx, &PyList_Type); + list = sym_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1149,7 +1149,7 @@ case _BUILD_SET: { JitOptRef set; - set = ref_new_type(ctx, &PySet_Type); + set = sym_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1158,7 +1158,7 @@ case _BUILD_MAP: { JitOptRef map; - map = ref_new_type(ctx, &PyDict_Type); + map = sym_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); @@ -1189,7 +1189,7 @@ case _LOAD_SUPER_ATTR_ATTR: { JitOptRef attr_st; - attr_st = ref_new_not_null(ctx); + attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1199,8 +1199,8 @@ case _LOAD_SUPER_ATTR_METHOD: { JitOptRef attr; JitOptRef self_or_null; - attr = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1215,9 +1215,9 @@ owner = stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); if (oparg & 1) { - self_or_null[0] = ref_new_unknown(ctx); + self_or_null[0] = sym_new_unknown(ctx); } stack_pointer[-1] = attr; stack_pointer += (oparg&1); @@ -1230,12 +1230,12 @@ owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); - if (ref_matches_type_version(owner, type_version)) { + if (sym_matches_type_version(owner, type_version)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { PyTypeObject *type = _PyType_LookupByVersion(type_version); if (type) { - if (ref_set_type_version(owner, type_version)) { + if (sym_set_type_version(owner, type_version)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); } @@ -1255,7 +1255,7 @@ case _LOAD_ATTR_INSTANCE_VALUE: { JitOptRef attr; uint16_t offset = (uint16_t)this_instr->operand0; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)offset; stack_pointer[-1] = attr; break; @@ -1270,8 +1270,8 @@ (void)dict_version; (void)index; attr = PyJitRef_NULL; - if (ref_is_const(ctx, owner)) { - PyModuleObject *mod = (PyModuleObject *)ref_get_const(ctx, owner); + if (sym_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; stack_pointer[-1] = attr; @@ -1280,12 +1280,12 @@ PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = ref_new_const(ctx, res); + attr = sym_new_const(ctx, res); } } } if (PyJitRef_IsNull(attr)) { - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); } stack_pointer[-1] = attr; break; @@ -1294,7 +1294,7 @@ case _LOAD_ATTR_WITH_HINT: { JitOptRef attr; uint16_t hint = (uint16_t)this_instr->operand0; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)hint; stack_pointer[-1] = attr; break; @@ -1303,7 +1303,7 @@ case _LOAD_ATTR_SLOT: { JitOptRef attr; uint16_t index = (uint16_t)this_instr->operand0; - attr = ref_new_not_null(ctx); + attr = sym_new_not_null(ctx); (void)index; stack_pointer[-1] = attr; break; @@ -1315,11 +1315,11 @@ uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { - if (type == ref_get_const(ctx, owner)) { + if (type == sym_get_const(ctx, owner)) { REPLACE_OP(this_instr, _NOP, 0, 0); } else { - ref_set_const(owner, type); + sym_set_const(owner, type); } } break; @@ -1331,7 +1331,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = (PyTypeObject *)ref_get_const(ctx, owner); + PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1377,10 +1377,10 @@ case _COMPARE_OP: { JitOptRef res; if (oparg & 16) { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); } else { - res = _Py_uop_ref_new_not_null(ctx); + res = _Py_uop_sym_new_not_null(ctx); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1390,7 +1390,7 @@ case _COMPARE_OP_FLOAT: { JitOptRef res; - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1403,11 +1403,11 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (ref_is_const(ctx, left) && ref_is_const(ctx, right)) { - assert(PyLong_CheckExact(ref_get_const(ctx, left))); - assert(PyLong_CheckExact(ref_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(ref_get_const(ctx, left), - ref_get_const(ctx, right), + if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), + sym_get_const(ctx, right), oparg >> 5); if (tmp == NULL) { goto error; @@ -1415,14 +1415,14 @@ assert(PyBool_Check(tmp)); assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = ref_new_const(ctx, tmp); + res = sym_new_const(ctx, tmp); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(tmp); } else { - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer += -1; } stack_pointer[-1] = res; @@ -1431,7 +1431,7 @@ case _COMPARE_OP_STR: { JitOptRef res; - res = ref_new_type(ctx, &PyBool_Type); + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1440,7 +1440,7 @@ case _IS_OP: { JitOptRef b; - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1449,7 +1449,7 @@ case _CONTAINS_OP: { JitOptRef b; - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1459,8 +1459,8 @@ case _GUARD_TOS_ANY_SET: { JitOptRef tos; tos = stack_pointer[-1]; - if (ref_matches_type(tos, &PySet_Type) || - ref_matches_type(tos, &PyFrozenSet_Type)) + if (sym_matches_type(tos, &PySet_Type) || + sym_matches_type(tos, &PyFrozenSet_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } @@ -1469,7 +1469,7 @@ case _CONTAINS_OP_SET: { JitOptRef b; - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1478,7 +1478,7 @@ case _CONTAINS_OP_DICT: { JitOptRef b; - b = ref_new_type(ctx, &PyBool_Type); + b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1488,8 +1488,8 @@ case _CHECK_EG_MATCH: { JitOptRef rest; JitOptRef match; - rest = ref_new_not_null(ctx); - match = ref_new_not_null(ctx); + rest = sym_new_not_null(ctx); + match = sym_new_not_null(ctx); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; @@ -1497,14 +1497,14 @@ case _CHECK_EXC_MATCH: { JitOptRef b; - b = ref_new_not_null(ctx); + b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1513,7 +1513,7 @@ case _IMPORT_FROM: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1526,7 +1526,7 @@ case _IS_NONE: { JitOptRef b; - b = ref_new_not_null(ctx); + b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } @@ -1535,9 +1535,9 @@ JitOptRef obj; JitOptRef len; obj = stack_pointer[-1]; - int tuple_length = ref_tuple_length(obj); + int tuple_length = sym_tuple_length(obj); if (tuple_length == -1) { - len = ref_new_type(ctx, &PyLong_Type); + len = sym_new_type(ctx, &PyLong_Type); } else { assert(tuple_length >= 0); @@ -1548,7 +1548,7 @@ if (_Py_IsImmortal(temp)) { REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } - len = ref_new_const(ctx, temp); + len = sym_new_const(ctx, temp); stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1563,7 +1563,7 @@ case _MATCH_CLASS: { JitOptRef attrs; - attrs = ref_new_not_null(ctx); + attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1572,7 +1572,7 @@ case _MATCH_MAPPING: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1581,7 +1581,7 @@ case _MATCH_SEQUENCE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1590,7 +1590,7 @@ case _MATCH_KEYS: { JitOptRef values_or_none; - values_or_none = ref_new_not_null(ctx); + values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1600,8 +1600,8 @@ case _GET_ITER: { JitOptRef iter; JitOptRef index_or_null; - iter = ref_new_not_null(ctx); - index_or_null = ref_new_not_null(ctx); + iter = sym_new_not_null(ctx); + index_or_null = sym_new_not_null(ctx); stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; @@ -1611,7 +1611,7 @@ case _GET_YIELD_FROM_ITER: { JitOptRef iter; - iter = ref_new_not_null(ctx); + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -1620,7 +1620,7 @@ case _FOR_ITER_TIER_TWO: { JitOptRef next; - next = ref_new_not_null(ctx); + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1643,7 +1643,7 @@ case _ITER_NEXT_LIST_TIER_TWO: { JitOptRef next; - next = ref_new_not_null(ctx); + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1653,10 +1653,10 @@ case _ITER_CHECK_TUPLE: { JitOptRef iter; iter = stack_pointer[-2]; - if (ref_matches_type(iter, &PyTuple_Type)) { + if (sym_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_type(iter, &PyTuple_Type); + sym_set_type(iter, &PyTuple_Type); break; } @@ -1668,7 +1668,7 @@ case _ITER_NEXT_TUPLE: { JitOptRef next; - next = ref_new_not_null(ctx); + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1687,7 +1687,7 @@ case _ITER_NEXT_RANGE: { JitOptRef next; - next = ref_new_type(ctx, &PyLong_Type); + next = sym_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1709,7 +1709,7 @@ JitOptRef *method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; - method_and_self[0] = ref_new_null(ctx); + method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1719,14 +1719,14 @@ case _LOAD_SPECIAL: { JitOptRef *method_and_self; method_and_self = &stack_pointer[-2]; - method_and_self[0] = ref_new_not_null(ctx); - method_and_self[1] = ref_new_unknown(ctx); + method_and_self[0] = sym_new_not_null(ctx); + method_and_self[1] = sym_new_unknown(ctx); break; } case _WITH_EXCEPT_START: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1736,8 +1736,8 @@ case _PUSH_EXC_INFO: { JitOptRef prev_exc; JitOptRef new_exc; - prev_exc = ref_new_not_null(ctx); - new_exc = ref_new_not_null(ctx); + prev_exc = sym_new_not_null(ctx); + new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -1760,7 +1760,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1780,7 +1780,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1799,7 +1799,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1814,7 +1814,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, @@ -1834,7 +1834,7 @@ owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - PyTypeObject *type = ref_get_type(owner); + PyTypeObject *type = sym_get_type(owner); PyObject *name = PyTuple_GET_ITEM(co->co_names, oparg >> 1); attr = lookup_attr(ctx, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, @@ -1855,8 +1855,8 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; (void)args; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -1886,12 +1886,12 @@ JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; - if (ref_is_const(ctx, callable) && ref_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(ref_get_const(ctx, callable))); + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)ref_get_const(ctx, callable); + this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); } - ref_set_type(callable, &PyFunction_Type); + sym_set_type(callable, &PyFunction_Type); break; } @@ -1913,7 +1913,7 @@ case _CALL_NON_PY_GENERAL: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1925,8 +1925,8 @@ JitOptRef callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - ref_set_null(null); - ref_set_type(callable, &PyMethod_Type); + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); break; } @@ -1935,8 +1935,8 @@ JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -1954,12 +1954,12 @@ JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(ref_matches_type(callable, &PyFunction_Type)); - if (ref_is_const(ctx, callable)) { - if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)ref_get_const(ctx, callable); + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(ctx, callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + !ref_is_null(self_or_null)) { + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } } @@ -1993,11 +1993,11 @@ } assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); - if (ref_is_not_null(self_or_null)) { + if (sym_is_not_null(self_or_null)) { args--; argcount++; } - if (ref_is_null(self_or_null) || ref_is_not_null(self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { new_frame = frame_new(ctx, co, 0, args, argcount); } else { new_frame = frame_new(ctx, co, 0, NULL, 0); @@ -2043,40 +2043,40 @@ case _GUARD_NOS_NULL: { JitOptRef null; null = stack_pointer[-2]; - if (ref_is_null(null)) { + if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_null(null); + sym_set_null(null); break; } case _GUARD_NOS_NOT_NULL: { JitOptRef nos; nos = stack_pointer[-2]; - if (ref_is_not_null(nos)) { + if (sym_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_non_null(nos); + sym_set_non_null(nos); break; } case _GUARD_THIRD_NULL: { JitOptRef null; null = stack_pointer[-3]; - if (ref_is_null(null)) { + if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_null(null); + sym_set_null(null); break; } case _GUARD_CALLABLE_TYPE_1: { JitOptRef callable; callable = stack_pointer[-3]; - if (ref_get_const(ctx, callable) == (PyObject *)&PyType_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyType_Type); + sym_set_const(callable, (PyObject *)&PyType_Type); break; } @@ -2084,11 +2084,11 @@ JitOptRef arg; JitOptRef res; arg = stack_pointer[-1]; - if (ref_has_type(arg)) { - res = ref_new_const(ctx, (PyObject *)ref_get_type(arg)); + if (sym_has_type(arg)) { + res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); } else { - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2099,10 +2099,10 @@ case _GUARD_CALLABLE_STR_1: { JitOptRef callable; callable = stack_pointer[-3]; - if (ref_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyUnicode_Type); + sym_set_const(callable, (PyObject *)&PyUnicode_Type); break; } @@ -2110,11 +2110,11 @@ JitOptRef arg; JitOptRef res; arg = stack_pointer[-1]; - if (ref_matches_type(arg, &PyUnicode_Type)) { + if (sym_matches_type(arg, &PyUnicode_Type)) { res = arg; } else { - res = ref_new_type(ctx, &PyUnicode_Type); + res = sym_new_type(ctx, &PyUnicode_Type); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2125,10 +2125,10 @@ case _GUARD_CALLABLE_TUPLE_1: { JitOptRef callable; callable = stack_pointer[-3]; - if (ref_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { + if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, (PyObject *)&PyTuple_Type); + sym_set_const(callable, (PyObject *)&PyTuple_Type); break; } @@ -2136,11 +2136,11 @@ JitOptRef arg; JitOptRef res; arg = stack_pointer[-1]; - if (ref_matches_type(arg, &PyTuple_Type)) { + if (sym_matches_type(arg, &PyTuple_Type)) { res = arg; } else { - res = ref_new_type(ctx, &PyTuple_Type); + res = sym_new_type(ctx, &PyTuple_Type); } stack_pointer[-3] = res; stack_pointer += -2; @@ -2158,8 +2158,8 @@ uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; (void)args; - callable = ref_new_not_null(ctx); - self_or_null = ref_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; break; @@ -2183,7 +2183,7 @@ case _CALL_BUILTIN_CLASS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2192,7 +2192,7 @@ case _CALL_BUILTIN_O: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2201,7 +2201,7 @@ case _CALL_BUILTIN_FAST: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2210,7 +2210,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2221,16 +2221,16 @@ JitOptRef callable; callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; - if (ref_get_const(ctx, callable) == len) { + if (sym_get_const(ctx, callable) == len) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, len); + sym_set_const(callable, len); break; } case _CALL_LEN: { JitOptRef res; - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2241,10 +2241,10 @@ JitOptRef callable; callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; - if (ref_get_const(ctx, callable) == isinstance) { + if (sym_get_const(ctx, callable) == isinstance) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, isinstance); + sym_set_const(callable, isinstance); break; } @@ -2254,15 +2254,15 @@ JitOptRef res; cls = stack_pointer[-1]; instance = stack_pointer[-2]; - res = ref_new_type(ctx, &PyBool_Type); - PyTypeObject *inst_type = ref_get_type(instance); - PyTypeObject *cls_o = (PyTypeObject *)ref_get_const(ctx, cls); - if (inst_type && cls_o && ref_matches_type(cls, &PyType_Type)) { + res = sym_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = sym_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); + if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { PyObject *out = Py_False; if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { out = Py_True; } - ref_set_const(res, out); + sym_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } stack_pointer[-4] = res; @@ -2275,10 +2275,10 @@ JitOptRef callable; callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; - if (ref_get_const(ctx, callable) == list_append) { + if (sym_get_const(ctx, callable) == list_append) { REPLACE_OP(this_instr, _NOP, 0, 0); } - ref_set_const(callable, list_append); + sym_set_const(callable, list_append); break; } @@ -2290,7 +2290,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2299,7 +2299,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2308,7 +2308,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2317,7 +2317,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2360,7 +2360,7 @@ case _CALL_KW_NON_PY: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2375,14 +2375,14 @@ case _MAKE_FUNCTION: { JitOptRef func; - func = ref_new_not_null(ctx); + func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { JitOptRef func_out; - func_out = ref_new_not_null(ctx); + func_out = sym_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2394,7 +2394,7 @@ ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); assert(corresponding_check_stack == NULL); assert(co != NULL); int framesize = co->co_framesize; @@ -2413,7 +2413,7 @@ case _BUILD_SLICE: { JitOptRef slice; - slice = ref_new_type(ctx, &PySlice_Type); + slice = sym_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2422,21 +2422,21 @@ case _CONVERT_VALUE: { JitOptRef result; - result = ref_new_not_null(ctx); + result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { JitOptRef res; - res = ref_new_not_null(ctx); + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2461,38 +2461,38 @@ JitOptRef res; rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; - bool lhs_int = ref_matches_type(lhs, &PyLong_Type); - bool rhs_int = ref_matches_type(rhs, &PyLong_Type); - bool lhs_float = ref_matches_type(lhs, &PyFloat_Type); - bool rhs_float = ref_matches_type(rhs, &PyFloat_Type); + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { if (rhs_float) { - res = ref_new_unknown(ctx); + res = sym_new_unknown(ctx); } else if (lhs_float) { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } - else if (!ref_is_const(ctx, rhs)) { - res = ref_new_unknown(ctx); + else if (!sym_is_const(ctx, rhs)) { + res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)ref_get_const(ctx, rhs))) { - res = ref_new_type(ctx, &PyFloat_Type); + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { + res = sym_new_type(ctx, &PyFloat_Type); } else { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } } else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } else if (lhs_int && rhs_int) { - res = ref_new_type(ctx, &PyLong_Type); + res = sym_new_type(ctx, &PyLong_Type); } else { - res = ref_new_type(ctx, &PyFloat_Type); + res = sym_new_type(ctx, &PyFloat_Type); } stack_pointer[-2] = res; stack_pointer += -1; @@ -2535,12 +2535,12 @@ case _GUARD_IS_TRUE_POP: { JitOptRef flag; flag = stack_pointer[-1]; - if (ref_is_const(ctx, flag)) { - PyObject *value = ref_get_const(ctx, flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } - ref_set_const(flag, Py_True); + sym_set_const(flag, Py_True); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2549,12 +2549,12 @@ case _GUARD_IS_FALSE_POP: { JitOptRef flag; flag = stack_pointer[-1]; - if (ref_is_const(ctx, flag)) { - PyObject *value = ref_get_const(ctx, flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } - ref_set_const(flag, Py_False); + sym_set_const(flag, Py_False); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2563,16 +2563,16 @@ case _GUARD_IS_NONE_POP: { JitOptRef val; val = stack_pointer[-1]; - if (ref_is_const(ctx, val)) { - PyObject *value = ref_get_const(ctx, val); + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (ref_has_type(val)) { - assert(!ref_matches_type(val, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - ref_set_const(val, Py_None); + sym_set_const(val, Py_None); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2581,13 +2581,13 @@ case _GUARD_IS_NOT_NONE_POP: { JitOptRef val; val = stack_pointer[-1]; - if (ref_is_const(ctx, val)) { - PyObject *value = ref_get_const(ctx, val); + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (ref_has_type(val)) { - assert(!ref_matches_type(val, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } stack_pointer += -1; @@ -2629,7 +2629,7 @@ case _LOAD_CONST_INLINE: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2639,7 +2639,7 @@ case _POP_TOP_LOAD_CONST_INLINE: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } @@ -2647,7 +2647,7 @@ case _LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2675,14 +2675,14 @@ case _POP_TOP_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _POP_TWO_LOAD_CONST_INLINE_BORROW: { JitOptRef value; - value = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2692,7 +2692,7 @@ case _POP_CALL_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2702,7 +2702,7 @@ case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2712,7 +2712,7 @@ case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(ref_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -2722,8 +2722,8 @@ case _LOAD_CONST_UNDER_INLINE: { JitOptRef value; JitOptRef new; - value = ref_new_not_null(ctx); - new = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); + new = sym_new_not_null(ctx); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; @@ -2734,8 +2734,8 @@ case _LOAD_CONST_UNDER_INLINE_BORROW: { JitOptRef value; JitOptRef new; - value = ref_new_not_null(ctx); - new = ref_new_not_null(ctx); + value = sym_new_not_null(ctx); + new = sym_new_not_null(ctx); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 994f591c561d6c..e19c0938e20446 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -123,20 +123,20 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_ref_is_bottom(JitOptRef ref) +_Py_uop_sym_is_bottom(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_ref_is_not_null(JitOptRef ref) { +_Py_uop_sym_is_not_null(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { @@ -144,7 +144,7 @@ _Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef ref) } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_ref_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); if (truthiness < 0) { return false; } @@ -155,14 +155,14 @@ _Py_uop_ref_is_const(JitOptContext *ctx, JitOptRef ref) } bool -_Py_uop_ref_is_null(JitOptRef ref) +_Py_uop_sym_is_null(JitOptRef ref) { return PyJitRef_AsSymbolBorrow(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * -_Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { @@ -170,7 +170,7 @@ _Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef ref) } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_ref_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); if (truthiness < 0) { return NULL; } @@ -182,7 +182,7 @@ _Py_uop_ref_get_const(JitOptContext *ctx, JitOptRef ref) } void -_Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) +_Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; @@ -233,12 +233,12 @@ _Py_uop_ref_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) } bool -_Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) +_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); PyTypeObject *type = _PyType_LookupByVersion(version); if (type) { - _Py_uop_ref_set_type(ctx, ref, type); + _Py_uop_sym_set_type(ctx, ref, type); } JitSymType tag = sym->tag; switch(tag) { @@ -291,7 +291,7 @@ _Py_uop_ref_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int ver } void -_Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) +_Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; @@ -314,12 +314,12 @@ _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) return; case JIT_SYM_TUPLE_TAG: if (PyTuple_CheckExact(const_val)) { - Py_ssize_t len = _Py_uop_ref_tuple_length(ref); + Py_ssize_t len = _Py_uop_sym_tuple_length(ref); if (len == PyTuple_GET_SIZE(const_val)) { for (Py_ssize_t i = 0; i < len; i++) { - JitOptRef ref_item = _Py_uop_ref_tuple_getitem(ctx, ref, i); + JitOptRef sym_item = _Py_uop_sym_tuple_getitem(ctx, ref, i); PyObject *item = PyTuple_GET_ITEM(const_val, i); - _Py_uop_ref_set_const(ctx, ref_item, item); + _Py_uop_sym_set_const(ctx, sym_item, item); } make_const(sym, const_val); return; @@ -342,29 +342,29 @@ _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) return; case JIT_SYM_TRUTHINESS_TAG: if (!PyBool_Check(const_val) || - (_Py_uop_ref_is_const(ctx, ref) && - _Py_uop_ref_get_const(ctx, ref) != const_val)) + (_Py_uop_sym_is_const(ctx, ref) && + _Py_uop_sym_get_const(ctx, ref) != const_val)) { sym_set_bottom(ctx, sym); return; } JitOptRef value = PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->truthiness.value); - PyTypeObject *type = _Py_uop_ref_get_type(value); + PyTypeObject *type = _Py_uop_sym_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: if (type == &PyBool_Type) { - _Py_uop_ref_set_const(ctx, value, Py_True); + _Py_uop_sym_set_const(ctx, value, Py_True); } } // value is falsey: else if (type == &PyBool_Type) { - _Py_uop_ref_set_const(ctx, value, Py_False); + _Py_uop_sym_set_const(ctx, value, Py_False); } else if (type == &PyLong_Type) { - _Py_uop_ref_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_ZERO)); + _Py_uop_sym_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_ZERO)); } else if (type == &PyUnicode_Type) { - _Py_uop_ref_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_EMPTY_STR)); + _Py_uop_sym_set_const(ctx, value, Py_GetConstant(Py_CONSTANT_EMPTY_STR)); } // TODO: More types (GH-130415)! make_const(sym, const_val); @@ -373,7 +373,7 @@ _Py_uop_ref_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) } void -_Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { @@ -385,7 +385,7 @@ _Py_uop_ref_set_null(JitOptContext *ctx, JitOptRef ref) } void -_Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { @@ -397,7 +397,7 @@ _Py_uop_ref_set_non_null(JitOptContext *ctx, JitOptRef ref) } JitOptRef -_Py_uop_ref_new_unknown(JitOptContext *ctx) +_Py_uop_sym_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { @@ -407,7 +407,7 @@ _Py_uop_ref_new_unknown(JitOptContext *ctx) } JitOptRef -_Py_uop_ref_new_not_null(JitOptContext *ctx) +_Py_uop_sym_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { @@ -418,20 +418,20 @@ _Py_uop_ref_new_not_null(JitOptContext *ctx) } JitOptRef -_Py_uop_ref_new_type(JitOptContext *ctx, PyTypeObject *typ) +_Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { return out_of_space_ref(ctx); } JitOptRef ref = PyJitRef_FromSymbolSteal(res); - _Py_uop_ref_set_type(ctx, ref, typ); + _Py_uop_sym_set_type(ctx, ref, typ); return ref; } // Adds a new reference to const_val, owned by the symbol. JitOptRef -_Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) +_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); JitOptSymbol *res = sym_new(ctx); @@ -439,24 +439,24 @@ _Py_uop_ref_new_const(JitOptContext *ctx, PyObject *const_val) return out_of_space_ref(ctx); } JitOptRef ref = PyJitRef_FromSymbolSteal(res); - _Py_uop_ref_set_const(ctx, ref, const_val); + _Py_uop_sym_set_const(ctx, ref, const_val); return ref; } JitOptRef -_Py_uop_ref_new_null(JitOptContext *ctx) +_Py_uop_sym_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); if (null_sym == NULL) { return out_of_space_ref(ctx); } JitOptRef ref = PyJitRef_FromSymbolSteal(null_sym); - _Py_uop_ref_set_null(ctx, ref); + _Py_uop_sym_set_null(ctx, ref); return ref; } PyTypeObject * -_Py_uop_ref_get_type(JitOptRef ref) +_Py_uop_sym_get_type(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; @@ -481,7 +481,7 @@ _Py_uop_ref_get_type(JitOptRef ref) } unsigned int -_Py_uop_ref_get_type_version(JitOptRef ref) +_Py_uop_sym_get_type_version(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); JitSymType tag = sym->tag; @@ -506,26 +506,26 @@ _Py_uop_ref_get_type_version(JitOptRef ref) } bool -_Py_uop_ref_has_type(JitOptRef sym) +_Py_uop_sym_has_type(JitOptRef sym) { - return _Py_uop_ref_get_type(sym) != NULL; + return _Py_uop_sym_get_type(sym) != NULL; } bool -_Py_uop_ref_matches_type(JitOptRef sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - return _Py_uop_ref_get_type(sym) == typ; + return _Py_uop_sym_get_type(sym) == typ; } bool -_Py_uop_ref_matches_type_version(JitOptRef sym, unsigned int version) +_Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version) { - return _Py_uop_ref_get_type_version(sym) == version; + return _Py_uop_sym_get_type_version(sym) == version; } int -_Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) +_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); switch(sym->tag) { @@ -547,7 +547,7 @@ _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) case JIT_SYM_TRUTHINESS_TAG: ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_ref_truthiness(ctx, + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolBorrow(value)); if (truthiness < 0) { return truthiness; @@ -575,7 +575,7 @@ _Py_uop_ref_truthiness(JitOptContext *ctx, JitOptRef ref) } JitOptRef -_Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) +_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { @@ -596,24 +596,24 @@ _Py_uop_ref_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) } JitOptRef -_Py_uop_ref_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) +_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple) && item < PyTuple_GET_SIZE(tuple)) { - return _Py_uop_ref_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); + return _Py_uop_sym_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { return PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->tuple.items[item]); } - return _Py_uop_ref_new_not_null(ctx); + return _Py_uop_sym_new_not_null(ctx); } int -_Py_uop_ref_tuple_length(JitOptRef ref) +_Py_uop_sym_tuple_length(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { @@ -630,7 +630,7 @@ _Py_uop_ref_tuple_length(JitOptRef ref) // Return true if known to be immortal. bool -_Py_uop_sym_is_immortal(JitOptSymbol *sym) +_Py_uop_symbol_is_immortal(JitOptSymbol *sym) { if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return _Py_IsImmortal(sym->value.value); @@ -645,14 +645,14 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) } bool -_Py_uop_ref_is_immortal(JitOptRef ref) +_Py_uop_sym_is_immortal(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); - return _Py_uop_sym_is_immortal(sym); + return _Py_uop_symbol_is_immortal(sym); } JitOptRef -_Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) +_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref); // It's clearer to invert this in the signature: @@ -664,7 +664,7 @@ _Py_uop_ref_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) if (res == NULL) { return out_of_space_ref(ctx); } - int truthiness = _Py_uop_ref_truthiness(ctx, ref); + int truthiness = _Py_uop_sym_truthiness(ctx, ref); if (truthiness < 0) { res->tag = JIT_SYM_TRUTHINESS_TAG; res->truthiness.invert = invert; @@ -707,14 +707,14 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptRef local = _Py_uop_ref_new_unknown(ctx); + JitOptRef local = _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - JitOptRef stackvar = _Py_uop_ref_new_unknown(ctx); + JitOptRef stackvar = _Py_uop_sym_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -798,44 +798,44 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - JitOptRef ref = _Py_uop_ref_new_unknown(ctx); + JitOptRef ref = _Py_uop_sym_new_unknown(ctx); if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "top is NULL"); - TEST_PREDICATE(!_Py_uop_ref_is_not_null(ref), "top is not NULL"); - TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "top is a constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "top as constant is not NULL"); - TEST_PREDICATE(!_Py_uop_ref_is_bottom(ref), "top is bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(ref), "top is bottom"); ref = PyJitRef_FromSymbolSteal(make_bottom(ctx)); if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "bottom is NULL is not false"); - TEST_PREDICATE(!_Py_uop_ref_is_not_null(ref), "bottom is not NULL is not false"); - TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); - TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "bottom isn't bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "bottom isn't bottom"); - ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "int is NULL"); - TEST_PREDICATE(_Py_uop_ref_is_not_null(ref), "int isn't not NULL"); - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "int isn't int"); - TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_ref_is_const(ctx, ref), "int is a constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "int as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "int as constant is not NULL"); - _Py_uop_ref_set_type(ctx, ref, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_ref_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(int and float) isn't bottom"); + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); assert(val_42 != NULL); @@ -845,86 +845,86 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(val_43 != NULL); assert(_Py_IsImmortal(val_43)); - ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_ref_set_const(ctx, ref, val_42); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 1, "bool(42) is not True"); - TEST_PREDICATE(!_Py_uop_ref_is_null(ref), "42 is NULL"); - TEST_PREDICATE(_Py_uop_ref_is_not_null(ref), "42 isn't not NULL"); - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "42 isn't an int"); - TEST_PREDICATE(!_Py_uop_ref_matches_type(ref, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); - TEST_PREDICATE(_Py_uop_ref_is_immortal(ref), "42 is not immortal"); + _Py_uop_sym_set_const(ctx, ref, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "42 is not immortal"); - _Py_uop_ref_set_type(ctx, ref, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); - _Py_uop_ref_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(42 and float) isn't bottom"); + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and float) isn't bottom"); - ref = _Py_uop_ref_new_type(ctx, &PyBool_Type); - TEST_PREDICATE(_Py_uop_ref_is_immortal(ref), "a bool is not immortal"); + ref = _Py_uop_sym_new_type(ctx, &PyBool_Type); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "a bool is not immortal"); - ref = _Py_uop_ref_new_type(ctx, &PyLong_Type); + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_ref_set_const(ctx, ref, val_42); - _Py_uop_ref_set_const(ctx, ref, val_43); // Should make it bottom - TEST_PREDICATE(_Py_uop_ref_is_bottom(ref), "(42 and 43) isn't bottom"); + _Py_uop_sym_set_const(ctx, ref, val_42); + _Py_uop_sym_set_const(ctx, ref, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and 43) isn't bottom"); - ref = _Py_uop_ref_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(None) is not False"); - ref = _Py_uop_ref_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(False) is not False"); - ref = _Py_uop_ref_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "bool(0) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(None) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(False) is not False"); + ref = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(0) is not False"); - JitOptRef i1 = _Py_uop_ref_new_type(ctx, &PyFloat_Type); - JitOptRef i2 = _Py_uop_ref_new_const(ctx, val_43); + JitOptRef i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); + JitOptRef i2 = _Py_uop_sym_new_const(ctx, val_43); JitOptRef array[2] = { i1, i2 }; - ref = _Py_uop_ref_new_tuple(ctx, 2, array); + ref = _Py_uop_sym_new_tuple(ctx, 2, array); TEST_PREDICATE( - _Py_uop_ref_matches_type(_Py_uop_ref_tuple_getitem(ctx, ref, 0), &PyFloat_Type), + _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, ref, 0), &PyFloat_Type), "tuple item does not match value used to create tuple" ); TEST_PREDICATE( - _Py_uop_ref_get_const(ctx, _Py_uop_ref_tuple_getitem(ctx, ref, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; tuple = _PyTuple_FromArray(pair, 2); - ref = _Py_uop_ref_new_const(ctx, tuple); + ref = _Py_uop_sym_new_const(ctx, tuple); TEST_PREDICATE( - _Py_uop_ref_get_const(ctx, _Py_uop_ref_tuple_getitem(ctx, ref, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); - ref = _Py_uop_ref_new_type(ctx, &PyTuple_Type); + ref = _Py_uop_sym_new_type(ctx, &PyTuple_Type); TEST_PREDICATE( - _Py_uop_ref_is_not_null(_Py_uop_ref_tuple_getitem(ctx, ref, 42)), + _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, ref, 42)), "Unknown tuple item is not narrowed to non-NULL" ); - JitOptRef value = _Py_uop_ref_new_type(ctx, &PyBool_Type); - ref = _Py_uop_ref_new_truthiness(ctx, value, false); - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == -1, "truthiness is not unknown"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref) == false, "truthiness is constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == NULL, "truthiness is not NULL"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, value) == false, "value is constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, value) == NULL, "value is not NULL"); - _Py_uop_ref_set_const(ctx, ref, Py_False); - TEST_PREDICATE(_Py_uop_ref_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_ref_truthiness(ctx, ref) == 0, "truthiness is not True"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, ref) == true, "truthiness is not constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, ref) == Py_False, "truthiness is not False"); - TEST_PREDICATE(_Py_uop_ref_is_const(ctx, value) == true, "value is not constant"); - TEST_PREDICATE(_Py_uop_ref_get_const(ctx, value) == Py_True, "value is not True"); + JitOptRef value = _Py_uop_sym_new_type(ctx, &PyBool_Type); + ref = _Py_uop_sym_new_truthiness(ctx, value, false); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == -1, "truthiness is not unknown"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == false, "truthiness is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "truthiness is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == false, "value is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == NULL, "value is not NULL"); + _Py_uop_sym_set_const(ctx, ref, Py_False); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "truthiness is not True"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == true, "truthiness is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == Py_False, "truthiness is not False"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 39ffc1d7e0cbd6..ed03c316a54ae6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -735,7 +735,7 @@ def visit(stmt: Stmt) -> None: continue #if not tkn.text.startswith(("Py", "_Py", "monitor")): # continue - if tkn.text.startswith(("sym_", "optimize_", "ref_", "PyJitRef")): + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): # Optimize functions continue if tkn.text.endswith("Check"): diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index dd8420b792f53e..95ccc39af1f4a0 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -124,15 +124,15 @@ def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: local.in_local = True if var.is_array(): if var.size == "1": - out.emit(f"{var.name}[0] = ref_new_not_null(ctx);\n") + out.emit(f"{var.name}[0] = sym_new_not_null(ctx);\n") else: out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = ref_new_not_null(ctx);\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") out.emit("}\n") elif var.name == "null": - out.emit(f"{var.name} = ref_new_null(ctx);\n") + out.emit(f"{var.name} = sym_new_null(ctx);\n") else: - out.emit(f"{var.name} = ref_new_not_null(ctx);\n") + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") class OptimizerEmitter(Emitter): From 2d779c4ded74b3287f6ea72ef8795e9dde126146 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 4 Jun 2025 23:11:37 +0800 Subject: [PATCH 21/24] rename jitref functions --- Include/internal/pycore_optimizer.h | 11 ++--- Python/optimizer_symbols.c | 65 +++++++++++++++-------------- 2 files changed, 36 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 88acea076dbc80..07c33f9183c6ca 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -230,24 +230,19 @@ typedef union { #define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~Py_TAG_REFCNT))) static inline JitOptSymbol * -PyJitRef_AsSymbolBorrow(JitOptRef ref) +PyJitRef_Unwrap(JitOptRef ref) { return JIT_BITS_TO_PTR_MASKED(ref); } bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym); -static inline JitOptRef -PyJitRef_FromSymbolBorrow(JitOptSymbol *sym) -{ - return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; -} static inline JitOptRef -PyJitRef_FromSymbolSteal(JitOptSymbol *sym) +PyJitRef_Wrap(JitOptSymbol *sym) { if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) { - return PyJitRef_FromSymbolBorrow(sym); + return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; } return (JitOptRef){.bits=(uintptr_t)sym}; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index e19c0938e20446..838f25ad288ab0 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -91,7 +91,7 @@ out_of_space(JitOptContext *ctx) JitOptRef out_of_space_ref(JitOptContext *ctx) { - return PyJitRef_FromSymbolSteal(out_of_space(ctx)); + return PyJitRef_Wrap(out_of_space(ctx)); } static JitOptSymbol * @@ -125,26 +125,26 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) bool _Py_uop_sym_is_bottom(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool _Py_uop_sym_is_not_null(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return true; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return false; } @@ -157,20 +157,20 @@ _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) bool _Py_uop_sym_is_null(JitOptRef ref) { - return PyJitRef_AsSymbolBorrow(ref)->tag == JIT_SYM_NULL_TAG; + return PyJitRef_Unwrap(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * _Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return sym->value.value; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_FromSymbolSteal(value)); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return NULL; } @@ -184,7 +184,7 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -235,7 +235,7 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); PyTypeObject *type = _PyType_LookupByVersion(version); if (type) { _Py_uop_sym_set_type(ctx, ref, type); @@ -293,7 +293,7 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int ver void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -348,7 +348,8 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) sym_set_bottom(ctx, sym); return; } - JitOptRef value = PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->truthiness.value); + JitOptRef value = PyJitRef_Wrap( + allocation_base(ctx) + sym->truthiness.value); PyTypeObject *type = _Py_uop_sym_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: @@ -375,7 +376,7 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NULL_TAG; } @@ -387,7 +388,7 @@ _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NON_NULL_TAG; } @@ -403,7 +404,7 @@ _Py_uop_sym_new_unknown(JitOptContext *ctx) if (res == NULL) { return out_of_space_ref(ctx); } - return PyJitRef_FromSymbolSteal(res); + return PyJitRef_Wrap(res); } JitOptRef @@ -414,7 +415,7 @@ _Py_uop_sym_new_not_null(JitOptContext *ctx) return out_of_space_ref(ctx); } res->tag = JIT_SYM_NON_NULL_TAG; - return PyJitRef_FromSymbolSteal(res); + return PyJitRef_Wrap(res); } JitOptRef @@ -424,7 +425,7 @@ _Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) if (res == NULL) { return out_of_space_ref(ctx); } - JitOptRef ref = PyJitRef_FromSymbolSteal(res); + JitOptRef ref = PyJitRef_Wrap(res); _Py_uop_sym_set_type(ctx, ref, typ); return ref; } @@ -438,7 +439,7 @@ _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) if (res == NULL) { return out_of_space_ref(ctx); } - JitOptRef ref = PyJitRef_FromSymbolSteal(res); + JitOptRef ref = PyJitRef_Wrap(res); _Py_uop_sym_set_const(ctx, ref, const_val); return ref; } @@ -450,7 +451,7 @@ _Py_uop_sym_new_null(JitOptContext *ctx) if (null_sym == NULL) { return out_of_space_ref(ctx); } - JitOptRef ref = PyJitRef_FromSymbolSteal(null_sym); + JitOptRef ref = PyJitRef_Wrap(null_sym); _Py_uop_sym_set_null(ctx, ref); return ref; } @@ -458,7 +459,7 @@ _Py_uop_sym_new_null(JitOptContext *ctx) PyTypeObject * _Py_uop_sym_get_type(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -483,7 +484,7 @@ _Py_uop_sym_get_type(JitOptRef ref) unsigned int _Py_uop_sym_get_type_version(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -527,7 +528,7 @@ _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version) int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); switch(sym->tag) { case JIT_SYM_NULL_TAG: case JIT_SYM_TYPE_VERSION_TAG: @@ -548,7 +549,7 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; int truthiness = _Py_uop_sym_truthiness(ctx, - PyJitRef_FromSymbolBorrow(value)); + PyJitRef_Wrap(value)); if (truthiness < 0) { return truthiness; } @@ -589,16 +590,16 @@ _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) res->tag = JIT_SYM_TUPLE_TAG; res->tuple.length = size; for (int i = 0; i < size; i++) { - res->tuple.items[i] = (uint16_t)(PyJitRef_AsSymbolBorrow(args[i]) - allocation_base(ctx)); + res->tuple.items[i] = (uint16_t)(PyJitRef_Unwrap(args[i]) - allocation_base(ctx)); } } - return PyJitRef_FromSymbolSteal(res); + return PyJitRef_Wrap(res); } JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; @@ -607,7 +608,7 @@ _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { - return PyJitRef_FromSymbolSteal(allocation_base(ctx) + sym->tuple.items[item]); + return PyJitRef_Wrap(allocation_base(ctx) + sym->tuple.items[item]); } return _Py_uop_sym_new_not_null(ctx); } @@ -615,7 +616,7 @@ _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) int _Py_uop_sym_tuple_length(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple)) { @@ -647,14 +648,14 @@ _Py_uop_symbol_is_immortal(JitOptSymbol *sym) bool _Py_uop_sym_is_immortal(JitOptRef ref) { - JitOptSymbol *sym = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return _Py_uop_symbol_is_immortal(sym); } JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { - JitOptSymbol *value = PyJitRef_AsSymbolBorrow(ref); + JitOptSymbol *value = PyJitRef_Unwrap(ref); // It's clearer to invert this in the signature: bool invert = !truthy; if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.invert == invert) { @@ -673,7 +674,7 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) else { make_const(res, (truthiness ^ invert) ? Py_True : Py_False); } - return PyJitRef_FromSymbolSteal(res); + return PyJitRef_Wrap(res); } // 0 on success, -1 on error. @@ -809,7 +810,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "top as constant is not NULL"); TEST_PREDICATE(!_Py_uop_sym_is_bottom(ref), "top is bottom"); - ref = PyJitRef_FromSymbolSteal(make_bottom(ctx)); + ref = PyJitRef_Wrap(make_bottom(ctx)); if (PyJitRef_IsNull(ref)) { goto fail; } From b74e160f9d0810732b470ce707615d2a2810ccbc Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 5 Jun 2025 01:33:37 +0800 Subject: [PATCH 22/24] Address review --- Include/internal/pycore_optimizer.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 07c33f9183c6ca..6fd6b876224b70 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -226,8 +226,9 @@ typedef union { uintptr_t bits; } JitOptRef; -#define JIT_BITS_TO_PTR(REF) ((JitOptSymbol *)((REF).bits)) -#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~Py_TAG_REFCNT))) +#define REF_IS_BORROWED 1 + +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED))) static inline JitOptSymbol * PyJitRef_Unwrap(JitOptRef ref) @@ -242,7 +243,7 @@ static inline JitOptRef PyJitRef_Wrap(JitOptSymbol *sym) { if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) { - return (JitOptRef){.bits=(uintptr_t)sym | Py_TAG_REFCNT}; + return (JitOptRef){.bits=(uintptr_t)sym | REF_IS_BORROWED}; } return (JitOptRef){.bits=(uintptr_t)sym}; } @@ -250,14 +251,10 @@ PyJitRef_Wrap(JitOptSymbol *sym) static inline JitOptRef PyJitRef_Borrow(JitOptRef ref) { - return (JitOptRef){ .bits = ref.bits | Py_TAG_REFCNT }; + return (JitOptRef){ .bits = ref.bits | REF_IS_BORROWED }; } -#ifndef Py_GIL_DISABLED -static const JitOptRef PyJitRef_NULL = {.bits = PyStackRef_NULL_BITS}; -#else -static const JitOptRef PyJitRef_NULL = {.bits = Py_TAG_DEFERRED}; -#endif +static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; static inline bool PyJitRef_IsNull(JitOptRef ref) @@ -268,7 +265,7 @@ PyJitRef_IsNull(JitOptRef ref) static inline int PyJitRef_IsBorrowed(JitOptRef ref) { - return (ref.bits & Py_TAG_REFCNT) == Py_TAG_REFCNT; + return (ref.bits & REF_IS_BORROWED) == REF_IS_BORROWED; } struct _Py_UOpsAbstractFrame { From 3ebcc2013124f2a3163934ec6928e97b10fcffbc Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 5 Jun 2025 01:35:02 +0800 Subject: [PATCH 23/24] Update comment --- Python/optimizer_bytecodes.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 7d65a9187b2497..4f8197db208d6e 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -296,7 +296,7 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } - // TODO (gh-134584): Move this to the optimizer generator. + // TODO (gh-134584): Refactor this to use another uop if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } @@ -320,7 +320,7 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } - // TODO (gh-134584): Move this to the optimizer generator. + // TODO (gh-134584): Refactor this to use another uop if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } @@ -344,7 +344,7 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } - // TODO (gh-134584): Move this to the optimizer generator. + // TODO (gh-134584): Refactor this to use another uop if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } From 914f1ff84c51042a4e010155aefb1c049f16aa5d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:43:19 +0800 Subject: [PATCH 24/24] Fix changes from upstream (no more casts) --- Include/internal/pycore_optimizer.h | 3 --- Python/optimizer_bytecodes.c | 20 ++++++++++---------- Python/optimizer_cases.c.h | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 6fd6b876224b70..b27341cb1dbbeb 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -242,9 +242,6 @@ bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym); static inline JitOptRef PyJitRef_Wrap(JitOptSymbol *sym) { - if (sym == NULL || _Py_uop_symbol_is_immortal(sym)) { - return (JitOptRef){.bits=(uintptr_t)sym | REF_IS_BORROWED}; - } return (JitOptRef){.bits=(uintptr_t)sym}; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 3d8d652bb5075e..07b05fc3323ece 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -341,7 +341,7 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -666,7 +666,7 @@ dummy_func(void) { op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -733,9 +733,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } } @@ -754,11 +754,11 @@ dummy_func(void) { break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -770,7 +770,7 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - init_frame = NULL; + init_frame = PyJitRef_NULL; ctx->done = true; } @@ -837,13 +837,13 @@ dummy_func(void) { } op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame)) { - gen_frame = NULL; + gen_frame = PyJitRef_NULL; /* We are about to hit the end of the trace */ ctx->done = true; } op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame)) { - gen_frame = NULL; + gen_frame = PyJitRef_NULL; // We are about to hit the end of the trace: ctx->done = true; } @@ -863,7 +863,7 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b86d73f39a362a..354331ef618f67 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -695,7 +695,7 @@ case _BINARY_OP_SUBSCR_INIT_CALL: { JitOptRef new_frame; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3] = new_frame; stack_pointer += -2; @@ -809,7 +809,7 @@ case _SEND_GEN_FRAME: { JitOptRef gen_frame; - gen_frame = NULL; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = gen_frame; break; @@ -1305,7 +1305,7 @@ JitOptRef new_frame; PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = new_frame; break; @@ -1665,7 +1665,7 @@ case _FOR_ITER_GEN_FRAME: { JitOptRef gen_frame; - gen_frame = NULL; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[0] = gen_frame; stack_pointer += 1; @@ -1844,7 +1844,7 @@ ctx->done = true; break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1977,9 +1977,9 @@ argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; @@ -1993,7 +1993,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); @@ -2149,7 +2149,7 @@ case _CREATE_INIT_FRAME: { JitOptRef init_frame; - init_frame = NULL; + init_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; @@ -2316,7 +2316,7 @@ case _PY_FRAME_KW: { JitOptRef new_frame; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy