diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d3674726997f6a..b27341cb1dbbeb 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_typedefs.h" // _PyInterpreterFrame #include "pycore_uop_ids.h" +#include "pycore_stackref.h" #include @@ -220,15 +221,58 @@ typedef union _jit_opt_symbol { } JitOptSymbol; +// This mimics the _PyStackRef API +typedef union { + uintptr_t bits; +} JitOptRef; + +#define REF_IS_BORROWED 1 + +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED))) + +static inline JitOptSymbol * +PyJitRef_Unwrap(JitOptRef ref) +{ + return JIT_BITS_TO_PTR_MASKED(ref); +} + +bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym); + + +static inline JitOptRef +PyJitRef_Wrap(JitOptSymbol *sym) +{ + return (JitOptRef){.bits=(uintptr_t)sym}; +} + +static inline JitOptRef +PyJitRef_Borrow(JitOptRef ref) +{ + return (JitOptRef){ .bits = ref.bits | REF_IS_BORROWED }; +} + +static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; + +static inline bool +PyJitRef_IsNull(JitOptRef ref) +{ + return ref.bits == PyJitRef_NULL.bits; +} + +static inline int +PyJitRef_IsBorrowed(JitOptRef ref) +{ + return (ref.bits & REF_IS_BORROWED) == REF_IS_BORROWED; +} struct _Py_UOpsAbstractFrame { // Max stacklen int stack_len; int locals_len; - JitOptSymbol **stack_pointer; - JitOptSymbol **stack; - JitOptSymbol **locals; + JitOptRef *stack_pointer; + JitOptRef *stack; + JitOptRef *locals; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -251,37 +295,36 @@ typedef struct _JitOptContext { // Arena for the symbolic types. ty_arena t_arena; - JitOptSymbol **n_consumed; - JitOptSymbol **limit; - JitOptSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + JitOptRef *n_consumed; + JitOptRef *limit; + JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; } JitOptContext; -extern bool _Py_uop_sym_is_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym); -extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx); -extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx); -extern JitOptSymbol *_Py_uop_sym_new_type( +extern bool _Py_uop_sym_is_null(JitOptRef sym); +extern bool _Py_uop_sym_is_not_null(JitOptRef sym); +extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef sym); +extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_unknown(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_not_null(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_type( JitOptContext *ctx, PyTypeObject *typ); -extern JitOptSymbol *_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); -extern JitOptSymbol *_Py_uop_sym_new_null(JitOptContext *ctx); -extern bool _Py_uop_sym_has_type(JitOptSymbol *sym); -extern bool _Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val); -extern bool _Py_uop_sym_is_bottom(JitOptSymbol *sym); -extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym); -extern PyTypeObject *_Py_uop_sym_get_type(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args); -extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item); -extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); +extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); +extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); +extern bool _Py_uop_sym_has_type(JitOptRef sym); +extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version); +extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef sym, unsigned int version); +extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(JitOptRef sym); +extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef sym); +extern PyTypeObject *_Py_uop_sym_get_type(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); +extern JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef sym, int item); +extern int _Py_uop_sym_tuple_length(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); @@ -290,7 +333,7 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len); extern int _Py_uop_frame_pop(JitOptContext *ctx); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8211c5d056535e..f67626ec5c6ac5 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,22 +13,25 @@ extern "C" { #define _SET_IP 301 #define _BINARY_OP 302 #define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_EXTEND 306 -#define _BINARY_OP_INPLACE_ADD_UNICODE 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_MULTIPLY_INT 309 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 310 -#define _BINARY_OP_SUBSCR_DICT 311 -#define _BINARY_OP_SUBSCR_INIT_CALL 312 -#define _BINARY_OP_SUBSCR_LIST_INT 313 -#define _BINARY_OP_SUBSCR_LIST_SLICE 314 -#define _BINARY_OP_SUBSCR_STR_INT 315 -#define _BINARY_OP_SUBSCR_TUPLE_INT 316 -#define _BINARY_OP_SUBTRACT_FLOAT 317 -#define _BINARY_OP_SUBTRACT_INT 318 -#define _BINARY_SLICE 319 +#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS 304 +#define _BINARY_OP_ADD_INT 305 +#define _BINARY_OP_ADD_UNICODE 306 +#define _BINARY_OP_EXTEND 307 +#define _BINARY_OP_INPLACE_ADD_UNICODE 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS 310 +#define _BINARY_OP_MULTIPLY_INT 311 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 312 +#define _BINARY_OP_SUBSCR_DICT 313 +#define _BINARY_OP_SUBSCR_INIT_CALL 314 +#define _BINARY_OP_SUBSCR_LIST_INT 315 +#define _BINARY_OP_SUBSCR_LIST_SLICE 316 +#define _BINARY_OP_SUBSCR_STR_INT 317 +#define _BINARY_OP_SUBSCR_TUPLE_INT 318 +#define _BINARY_OP_SUBTRACT_FLOAT 319 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 320 +#define _BINARY_OP_SUBTRACT_INT 321 +#define _BINARY_SLICE 322 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -37,138 +40,138 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 320 -#define _CALL_BUILTIN_FAST 321 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 322 -#define _CALL_BUILTIN_O 323 +#define _CALL_BUILTIN_CLASS 323 +#define _CALL_BUILTIN_FAST 324 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 +#define _CALL_BUILTIN_O 326 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 324 -#define _CALL_KW_NON_PY 325 -#define _CALL_LEN 326 -#define _CALL_LIST_APPEND 327 -#define _CALL_METHOD_DESCRIPTOR_FAST 328 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 329 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 330 -#define _CALL_METHOD_DESCRIPTOR_O 331 -#define _CALL_NON_PY_GENERAL 332 -#define _CALL_STR_1 333 -#define _CALL_TUPLE_1 334 -#define _CALL_TYPE_1 335 -#define _CHECK_AND_ALLOCATE_OBJECT 336 -#define _CHECK_ATTR_CLASS 337 -#define _CHECK_ATTR_METHOD_LAZY_DICT 338 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 339 +#define _CALL_ISINSTANCE 327 +#define _CALL_KW_NON_PY 328 +#define _CALL_LEN 329 +#define _CALL_LIST_APPEND 330 +#define _CALL_METHOD_DESCRIPTOR_FAST 331 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 +#define _CALL_METHOD_DESCRIPTOR_O 334 +#define _CALL_NON_PY_GENERAL 335 +#define _CALL_STR_1 336 +#define _CALL_TUPLE_1 337 +#define _CALL_TYPE_1 338 +#define _CHECK_AND_ALLOCATE_OBJECT 339 +#define _CHECK_ATTR_CLASS 340 +#define _CHECK_ATTR_METHOD_LAZY_DICT 341 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 340 -#define _CHECK_FUNCTION_EXACT_ARGS 341 -#define _CHECK_FUNCTION_VERSION 342 -#define _CHECK_FUNCTION_VERSION_INLINE 343 -#define _CHECK_FUNCTION_VERSION_KW 344 -#define _CHECK_IS_NOT_PY_CALLABLE 345 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 346 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 347 -#define _CHECK_METHOD_VERSION 348 -#define _CHECK_METHOD_VERSION_KW 349 -#define _CHECK_PEP_523 350 -#define _CHECK_PERIODIC 351 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 352 -#define _CHECK_RECURSION_REMAINING 353 -#define _CHECK_STACK_SPACE 354 -#define _CHECK_STACK_SPACE_OPERAND 355 -#define _CHECK_VALIDITY 356 -#define _COMPARE_OP 357 -#define _COMPARE_OP_FLOAT 358 -#define _COMPARE_OP_INT 359 -#define _COMPARE_OP_STR 360 -#define _CONTAINS_OP 361 -#define _CONTAINS_OP_DICT 362 -#define _CONTAINS_OP_SET 363 +#define _CHECK_FUNCTION 343 +#define _CHECK_FUNCTION_EXACT_ARGS 344 +#define _CHECK_FUNCTION_VERSION 345 +#define _CHECK_FUNCTION_VERSION_INLINE 346 +#define _CHECK_FUNCTION_VERSION_KW 347 +#define _CHECK_IS_NOT_PY_CALLABLE 348 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 +#define _CHECK_METHOD_VERSION 351 +#define _CHECK_METHOD_VERSION_KW 352 +#define _CHECK_PEP_523 353 +#define _CHECK_PERIODIC 354 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 +#define _CHECK_RECURSION_REMAINING 356 +#define _CHECK_STACK_SPACE 357 +#define _CHECK_STACK_SPACE_OPERAND 358 +#define _CHECK_VALIDITY 359 +#define _COMPARE_OP 360 +#define _COMPARE_OP_FLOAT 361 +#define _COMPARE_OP_INT 362 +#define _COMPARE_OP_STR 363 +#define _CONTAINS_OP 364 +#define _CONTAINS_OP_DICT 365 +#define _CONTAINS_OP_SET 366 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 364 -#define _COPY_1 365 -#define _COPY_2 366 -#define _COPY_3 367 +#define _COPY 367 +#define _COPY_1 368 +#define _COPY_2 369 +#define _COPY_3 370 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 368 +#define _CREATE_INIT_FRAME 371 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 369 +#define _DEOPT 372 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 370 -#define _DO_CALL_FUNCTION_EX 371 -#define _DO_CALL_KW 372 +#define _DO_CALL 373 +#define _DO_CALL_FUNCTION_EX 374 +#define _DO_CALL_KW 375 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 373 +#define _ERROR_POP_N 376 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 374 -#define _EXPAND_METHOD_KW 375 -#define _FATAL_ERROR 376 +#define _EXPAND_METHOD 377 +#define _EXPAND_METHOD_KW 378 +#define _FATAL_ERROR 379 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 377 -#define _FOR_ITER_GEN_FRAME 378 -#define _FOR_ITER_TIER_TWO 379 +#define _FOR_ITER 380 +#define _FOR_ITER_GEN_FRAME 381 +#define _FOR_ITER_TIER_TWO 382 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 380 -#define _GUARD_CALLABLE_ISINSTANCE 381 -#define _GUARD_CALLABLE_LEN 382 -#define _GUARD_CALLABLE_LIST_APPEND 383 -#define _GUARD_CALLABLE_STR_1 384 -#define _GUARD_CALLABLE_TUPLE_1 385 -#define _GUARD_CALLABLE_TYPE_1 386 -#define _GUARD_DORV_NO_DICT 387 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 388 -#define _GUARD_GLOBALS_VERSION 389 -#define _GUARD_IS_FALSE_POP 390 -#define _GUARD_IS_NONE_POP 391 -#define _GUARD_IS_NOT_NONE_POP 392 -#define _GUARD_IS_TRUE_POP 393 -#define _GUARD_KEYS_VERSION 394 -#define _GUARD_NOS_DICT 395 -#define _GUARD_NOS_FLOAT 396 -#define _GUARD_NOS_INT 397 -#define _GUARD_NOS_LIST 398 -#define _GUARD_NOS_NOT_NULL 399 -#define _GUARD_NOS_NULL 400 -#define _GUARD_NOS_TUPLE 401 -#define _GUARD_NOS_UNICODE 402 -#define _GUARD_NOT_EXHAUSTED_LIST 403 -#define _GUARD_NOT_EXHAUSTED_RANGE 404 -#define _GUARD_NOT_EXHAUSTED_TUPLE 405 -#define _GUARD_THIRD_NULL 406 -#define _GUARD_TOS_ANY_SET 407 -#define _GUARD_TOS_DICT 408 -#define _GUARD_TOS_FLOAT 409 -#define _GUARD_TOS_INT 410 -#define _GUARD_TOS_LIST 411 -#define _GUARD_TOS_SLICE 412 -#define _GUARD_TOS_TUPLE 413 -#define _GUARD_TOS_UNICODE 414 -#define _GUARD_TYPE_VERSION 415 -#define _GUARD_TYPE_VERSION_AND_LOCK 416 +#define _GUARD_BINARY_OP_EXTEND 383 +#define _GUARD_CALLABLE_ISINSTANCE 384 +#define _GUARD_CALLABLE_LEN 385 +#define _GUARD_CALLABLE_LIST_APPEND 386 +#define _GUARD_CALLABLE_STR_1 387 +#define _GUARD_CALLABLE_TUPLE_1 388 +#define _GUARD_CALLABLE_TYPE_1 389 +#define _GUARD_DORV_NO_DICT 390 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 391 +#define _GUARD_GLOBALS_VERSION 392 +#define _GUARD_IS_FALSE_POP 393 +#define _GUARD_IS_NONE_POP 394 +#define _GUARD_IS_NOT_NONE_POP 395 +#define _GUARD_IS_TRUE_POP 396 +#define _GUARD_KEYS_VERSION 397 +#define _GUARD_NOS_DICT 398 +#define _GUARD_NOS_FLOAT 399 +#define _GUARD_NOS_INT 400 +#define _GUARD_NOS_LIST 401 +#define _GUARD_NOS_NOT_NULL 402 +#define _GUARD_NOS_NULL 403 +#define _GUARD_NOS_TUPLE 404 +#define _GUARD_NOS_UNICODE 405 +#define _GUARD_NOT_EXHAUSTED_LIST 406 +#define _GUARD_NOT_EXHAUSTED_RANGE 407 +#define _GUARD_NOT_EXHAUSTED_TUPLE 408 +#define _GUARD_THIRD_NULL 409 +#define _GUARD_TOS_ANY_SET 410 +#define _GUARD_TOS_DICT 411 +#define _GUARD_TOS_FLOAT 412 +#define _GUARD_TOS_INT 413 +#define _GUARD_TOS_LIST 414 +#define _GUARD_TOS_SLICE 415 +#define _GUARD_TOS_TUPLE 416 +#define _GUARD_TOS_UNICODE 417 +#define _GUARD_TYPE_VERSION 418 +#define _GUARD_TYPE_VERSION_AND_LOCK 419 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 417 -#define _INIT_CALL_PY_EXACT_ARGS 418 -#define _INIT_CALL_PY_EXACT_ARGS_0 419 -#define _INIT_CALL_PY_EXACT_ARGS_1 420 -#define _INIT_CALL_PY_EXACT_ARGS_2 421 -#define _INIT_CALL_PY_EXACT_ARGS_3 422 -#define _INIT_CALL_PY_EXACT_ARGS_4 423 -#define _INSERT_NULL 424 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 420 +#define _INIT_CALL_PY_EXACT_ARGS 421 +#define _INIT_CALL_PY_EXACT_ARGS_0 422 +#define _INIT_CALL_PY_EXACT_ARGS_1 423 +#define _INIT_CALL_PY_EXACT_ARGS_2 424 +#define _INIT_CALL_PY_EXACT_ARGS_3 425 +#define _INIT_CALL_PY_EXACT_ARGS_4 426 +#define _INSERT_NULL 427 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -178,173 +181,173 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 425 +#define _IS_NONE 428 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 426 -#define _ITER_CHECK_RANGE 427 -#define _ITER_CHECK_TUPLE 428 -#define _ITER_JUMP_LIST 429 -#define _ITER_JUMP_RANGE 430 -#define _ITER_JUMP_TUPLE 431 -#define _ITER_NEXT_LIST 432 -#define _ITER_NEXT_LIST_TIER_TWO 433 -#define _ITER_NEXT_RANGE 434 -#define _ITER_NEXT_TUPLE 435 -#define _JUMP_TO_TOP 436 +#define _ITER_CHECK_LIST 429 +#define _ITER_CHECK_RANGE 430 +#define _ITER_CHECK_TUPLE 431 +#define _ITER_JUMP_LIST 432 +#define _ITER_JUMP_RANGE 433 +#define _ITER_JUMP_TUPLE 434 +#define _ITER_NEXT_LIST 435 +#define _ITER_NEXT_LIST_TIER_TWO 436 +#define _ITER_NEXT_RANGE 437 +#define _ITER_NEXT_TUPLE 438 +#define _JUMP_TO_TOP 439 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 437 -#define _LOAD_ATTR_CLASS 438 +#define _LOAD_ATTR 440 +#define _LOAD_ATTR_CLASS 441 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 439 -#define _LOAD_ATTR_METHOD_LAZY_DICT 440 -#define _LOAD_ATTR_METHOD_NO_DICT 441 -#define _LOAD_ATTR_METHOD_WITH_VALUES 442 -#define _LOAD_ATTR_MODULE 443 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 444 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 445 -#define _LOAD_ATTR_PROPERTY_FRAME 446 -#define _LOAD_ATTR_SLOT 447 -#define _LOAD_ATTR_WITH_HINT 448 +#define _LOAD_ATTR_INSTANCE_VALUE 442 +#define _LOAD_ATTR_METHOD_LAZY_DICT 443 +#define _LOAD_ATTR_METHOD_NO_DICT 444 +#define _LOAD_ATTR_METHOD_WITH_VALUES 445 +#define _LOAD_ATTR_MODULE 446 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 447 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 448 +#define _LOAD_ATTR_PROPERTY_FRAME 449 +#define _LOAD_ATTR_SLOT 450 +#define _LOAD_ATTR_WITH_HINT 451 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 449 +#define _LOAD_BYTECODE 452 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 450 -#define _LOAD_CONST_INLINE_BORROW 451 -#define _LOAD_CONST_UNDER_INLINE 452 -#define _LOAD_CONST_UNDER_INLINE_BORROW 453 +#define _LOAD_CONST_INLINE 453 +#define _LOAD_CONST_INLINE_BORROW 454 +#define _LOAD_CONST_UNDER_INLINE 455 +#define _LOAD_CONST_UNDER_INLINE_BORROW 456 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 454 -#define _LOAD_FAST_0 455 -#define _LOAD_FAST_1 456 -#define _LOAD_FAST_2 457 -#define _LOAD_FAST_3 458 -#define _LOAD_FAST_4 459 -#define _LOAD_FAST_5 460 -#define _LOAD_FAST_6 461 -#define _LOAD_FAST_7 462 +#define _LOAD_FAST 457 +#define _LOAD_FAST_0 458 +#define _LOAD_FAST_1 459 +#define _LOAD_FAST_2 460 +#define _LOAD_FAST_3 461 +#define _LOAD_FAST_4 462 +#define _LOAD_FAST_5 463 +#define _LOAD_FAST_6 464 +#define _LOAD_FAST_7 465 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 463 -#define _LOAD_FAST_BORROW_0 464 -#define _LOAD_FAST_BORROW_1 465 -#define _LOAD_FAST_BORROW_2 466 -#define _LOAD_FAST_BORROW_3 467 -#define _LOAD_FAST_BORROW_4 468 -#define _LOAD_FAST_BORROW_5 469 -#define _LOAD_FAST_BORROW_6 470 -#define _LOAD_FAST_BORROW_7 471 +#define _LOAD_FAST_BORROW 466 +#define _LOAD_FAST_BORROW_0 467 +#define _LOAD_FAST_BORROW_1 468 +#define _LOAD_FAST_BORROW_2 469 +#define _LOAD_FAST_BORROW_3 470 +#define _LOAD_FAST_BORROW_4 471 +#define _LOAD_FAST_BORROW_5 472 +#define _LOAD_FAST_BORROW_6 473 +#define _LOAD_FAST_BORROW_7 474 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 472 -#define _LOAD_GLOBAL_BUILTINS 473 -#define _LOAD_GLOBAL_MODULE 474 +#define _LOAD_GLOBAL 475 +#define _LOAD_GLOBAL_BUILTINS 476 +#define _LOAD_GLOBAL_MODULE 477 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 475 -#define _LOAD_SMALL_INT_0 476 -#define _LOAD_SMALL_INT_1 477 -#define _LOAD_SMALL_INT_2 478 -#define _LOAD_SMALL_INT_3 479 -#define _LOAD_SPECIAL 480 +#define _LOAD_SMALL_INT 478 +#define _LOAD_SMALL_INT_0 479 +#define _LOAD_SMALL_INT_1 480 +#define _LOAD_SMALL_INT_2 481 +#define _LOAD_SMALL_INT_3 482 +#define _LOAD_SPECIAL 483 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 481 +#define _MAKE_CALLARGS_A_TUPLE 484 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 482 +#define _MAKE_WARM 485 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 483 -#define _MAYBE_EXPAND_METHOD_KW 484 -#define _MONITOR_CALL 485 -#define _MONITOR_CALL_KW 486 -#define _MONITOR_JUMP_BACKWARD 487 -#define _MONITOR_RESUME 488 +#define _MAYBE_EXPAND_METHOD 486 +#define _MAYBE_EXPAND_METHOD_KW 487 +#define _MONITOR_CALL 488 +#define _MONITOR_CALL_KW 489 +#define _MONITOR_JUMP_BACKWARD 490 +#define _MONITOR_RESUME 491 #define _NOP NOP -#define _POP_CALL 489 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 490 -#define _POP_CALL_ONE 491 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 492 -#define _POP_CALL_TWO 493 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 494 +#define _POP_CALL 492 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 493 +#define _POP_CALL_ONE 494 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 495 +#define _POP_CALL_TWO 496 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 497 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 495 -#define _POP_JUMP_IF_TRUE 496 +#define _POP_JUMP_IF_FALSE 498 +#define _POP_JUMP_IF_TRUE 499 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 497 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 498 -#define _POP_TWO 499 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 500 +#define _POP_TOP_LOAD_CONST_INLINE 500 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 501 +#define _POP_TWO 502 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 503 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 501 +#define _PUSH_FRAME 504 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 502 -#define _PY_FRAME_GENERAL 503 -#define _PY_FRAME_KW 504 -#define _QUICKEN_RESUME 505 -#define _REPLACE_WITH_TRUE 506 +#define _PUSH_NULL_CONDITIONAL 505 +#define _PY_FRAME_GENERAL 506 +#define _PY_FRAME_KW 507 +#define _QUICKEN_RESUME 508 +#define _REPLACE_WITH_TRUE 509 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 507 -#define _SEND 508 -#define _SEND_GEN_FRAME 509 +#define _SAVE_RETURN_OFFSET 510 +#define _SEND 511 +#define _SEND_GEN_FRAME 512 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 510 -#define _STORE_ATTR 511 -#define _STORE_ATTR_INSTANCE_VALUE 512 -#define _STORE_ATTR_SLOT 513 -#define _STORE_ATTR_WITH_HINT 514 +#define _START_EXECUTOR 513 +#define _STORE_ATTR 514 +#define _STORE_ATTR_INSTANCE_VALUE 515 +#define _STORE_ATTR_SLOT 516 +#define _STORE_ATTR_WITH_HINT 517 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 515 -#define _STORE_FAST_0 516 -#define _STORE_FAST_1 517 -#define _STORE_FAST_2 518 -#define _STORE_FAST_3 519 -#define _STORE_FAST_4 520 -#define _STORE_FAST_5 521 -#define _STORE_FAST_6 522 -#define _STORE_FAST_7 523 +#define _STORE_FAST 518 +#define _STORE_FAST_0 519 +#define _STORE_FAST_1 520 +#define _STORE_FAST_2 521 +#define _STORE_FAST_3 522 +#define _STORE_FAST_4 523 +#define _STORE_FAST_5 524 +#define _STORE_FAST_6 525 +#define _STORE_FAST_7 526 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 524 -#define _STORE_SUBSCR 525 -#define _STORE_SUBSCR_DICT 526 -#define _STORE_SUBSCR_LIST_INT 527 -#define _SWAP 528 -#define _SWAP_2 529 -#define _SWAP_3 530 -#define _TIER2_RESUME_CHECK 531 -#define _TO_BOOL 532 +#define _STORE_SLICE 527 +#define _STORE_SUBSCR 528 +#define _STORE_SUBSCR_DICT 529 +#define _STORE_SUBSCR_LIST_INT 530 +#define _SWAP 531 +#define _SWAP_2 532 +#define _SWAP_3 533 +#define _TIER2_RESUME_CHECK 534 +#define _TO_BOOL 535 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 533 +#define _TO_BOOL_LIST 536 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 534 +#define _TO_BOOL_STR 537 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 535 -#define _UNPACK_SEQUENCE_LIST 536 -#define _UNPACK_SEQUENCE_TUPLE 537 -#define _UNPACK_SEQUENCE_TWO_TUPLE 538 +#define _UNPACK_SEQUENCE 538 +#define _UNPACK_SEQUENCE_LIST 539 +#define _UNPACK_SEQUENCE_TUPLE 540 +#define _UNPACK_SEQUENCE_TWO_TUPLE 541 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 538 +#define MAX_UOP_ID 541 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index fad87d4b586e64..40914c578c9038 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -94,6 +94,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -342,11 +345,13 @@ const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", @@ -356,6 +361,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SLICE] = "_BINARY_SLICE", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", @@ -799,6 +805,12 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: + return 2; + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: + return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 3fc2cb33795a5f..41833836e720ba 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -678,7 +678,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) def test_float_subtract_constant_propagation(self): def testfunc(n): @@ -700,7 +700,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", uops) def test_float_multiply_constant_propagation(self): def testfunc(n): @@ -722,7 +722,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", uops) def test_add_unicode_propagation(self): def testfunc(n): @@ -2262,6 +2262,20 @@ def f(n): self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + def test_float_op_refcount_elimination(self): + def testfunc(args): + a, b, n = args + c = 0.0 + for _ in range(n): + c += a + b + return c + + res, ex = self._run_with_optimizer(testfunc, (0.1, 0.1, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * (0.1 + 0.1)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) + def test_remove_guard_for_slice_list(self): def f(n): for i in range(n): diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 6411e4318b6c3a..9e0fd1218f2534 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1976,8 +1976,8 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - JitOptSymbol *arg1; - JitOptSymbol *out; + JitOptRef arg1; + JitOptRef out; arg1 = stack_pointer[-1]; out = EGGS(arg1); stack_pointer[-1] = out; @@ -1985,7 +1985,7 @@ def test_overridden_abstract_args(self): } case OP2: { - JitOptSymbol *out; + JitOptRef out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; @@ -2010,14 +2010,14 @@ def test_no_overridden_case(self): """ output = """ case OP: { - JitOptSymbol *out; + JitOptRef out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - JitOptSymbol *out; + JitOptRef out; out = NULL; stack_pointer[-1] = out; break; @@ -2151,7 +2151,7 @@ def test_validate_uop_unused_output(self): """ output = """ case OP: { - JitOptSymbol *foo; + JitOptRef foo; foo = NULL; stack_pointer[0] = foo; stack_pointer += 1; diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst new file mode 100644 index 00000000000000..5f9e1553ae7ca5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-23-14-54-07.gh-issue-134584.y-WDjf.rst @@ -0,0 +1 @@ +Add a reference count elimination pass to the JIT compiler. Patch by Ken Jin. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 27a04766cc8dd8..2dd1d27747a693 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -687,6 +687,52 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } + + pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + macro(BINARY_OP_MULTIPLY_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 74c78e4d1f5a69..3b58e7fab5553d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1063,6 +1063,87 @@ break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { _PyStackRef right; _PyStackRef left; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6a7df233819b9c..de337f637163ba 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -347,8 +347,8 @@ static int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, - JitOptSymbol *value, - JitOptSymbol **result_ptr) + JitOptRef value, + JitOptRef *result_ptr) { if (sym_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -375,7 +375,7 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) } } -static JitOptSymbol * +static JitOptRef lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, PyTypeObject *type, PyObject *name, uint16_t immortal, uint16_t mortal) @@ -440,6 +440,13 @@ get_code_with_logging(_PyUOpInstruction *op) return co; } +// TODO (gh-134584) generate most of this table automatically +const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { + [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, +}; + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( @@ -477,7 +484,7 @@ optimize_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptSymbol **stack_pointer = ctx->frame->stack_pointer; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG if (get_lltrace() >= 3) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 5a9a3a943a7b02..07b05fc3323ece 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -87,12 +87,12 @@ dummy_func(void) { } op(_LOAD_FAST_BORROW, (-- value)) { - value = GETLOCAL(oparg); + value = PyJitRef_Borrow(GETLOCAL(oparg)); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -251,6 +251,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { @@ -271,6 +275,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { @@ -291,6 +299,10 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyFloat_Type); } + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { @@ -310,7 +322,7 @@ dummy_func(void) { } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { - JitOptSymbol *res; + JitOptRef res; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); @@ -329,7 +341,7 @@ dummy_func(void) { } op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -488,7 +500,7 @@ dummy_func(void) { op(_LOAD_CONST, (-- value)) { PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { @@ -496,35 +508,35 @@ dummy_func(void) { assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -533,7 +545,7 @@ dummy_func(void) { } op(_SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -547,7 +559,7 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr)) { (void)dict_version; (void)index; - attr = NULL; + attr = PyJitRef_NULL; if (sym_is_const(ctx, owner)) { PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { @@ -561,7 +573,7 @@ dummy_func(void) { } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -654,7 +666,7 @@ dummy_func(void) { op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -712,7 +724,7 @@ dummy_func(void) { } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -721,9 +733,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } } @@ -742,11 +754,11 @@ dummy_func(void) { break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -758,12 +770,12 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - init_frame = NULL; + init_frame = PyJitRef_NULL; ctx->done = true; } op(_RETURN_VALUE, (retval -- res)) { - JitOptSymbol *temp = retval; + JitOptRef temp = retval; DEAD(retval); SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; @@ -825,13 +837,13 @@ dummy_func(void) { } op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame)) { - gen_frame = NULL; + gen_frame = PyJitRef_NULL; /* We are about to hit the end of the trace */ ctx->done = true; } op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame)) { - gen_frame = NULL; + gen_frame = PyJitRef_NULL; // We are about to hit the end of the trace: ctx->done = true; } @@ -851,7 +863,7 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8c4f0399c75a73..354331ef618f67 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -26,7 +26,7 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); if (sym_is_null(value)) { ctx->done = true; @@ -38,7 +38,7 @@ } case _LOAD_FAST: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; @@ -47,8 +47,8 @@ } case _LOAD_FAST_BORROW: { - JitOptSymbol *value; - value = GETLOCAL(oparg); + JitOptRef value; + value = PyJitRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -56,9 +56,9 @@ } case _LOAD_FAST_AND_CLEAR: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -67,10 +67,10 @@ } case _LOAD_CONST: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -78,12 +78,12 @@ } case _LOAD_SMALL_INT: { - JitOptSymbol *value; + JitOptRef value; PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -91,7 +91,7 @@ } case _STORE_FAST: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -112,7 +112,7 @@ } case _PUSH_NULL: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -133,7 +133,7 @@ } case _END_SEND: { - JitOptSymbol *val; + JitOptRef val; val = sym_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; @@ -142,15 +142,15 @@ } case _UNARY_NEGATIVE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; sym_set_type(value, &PyBool_Type); res = sym_new_truthiness(ctx, value, false); @@ -159,8 +159,8 @@ } case _TO_BOOL: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -171,7 +171,7 @@ } case _TO_BOOL_BOOL: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { @@ -183,8 +183,8 @@ } case _TO_BOOL_INT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -196,7 +196,7 @@ } case _GUARD_NOS_LIST: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -206,7 +206,7 @@ } case _GUARD_TOS_LIST: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -220,8 +220,8 @@ } case _TO_BOOL_LIST: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -232,8 +232,8 @@ } case _TO_BOOL_NONE: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -245,7 +245,7 @@ } case _GUARD_NOS_UNICODE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -255,7 +255,7 @@ } case _GUARD_TOS_UNICODE: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -265,8 +265,8 @@ } case _TO_BOOL_STR: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -277,7 +277,7 @@ } case _REPLACE_WITH_TRUE: { - JitOptSymbol *res; + JitOptRef res; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; @@ -285,8 +285,8 @@ } case _UNARY_INVERT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); @@ -299,7 +299,7 @@ } case _GUARD_NOS_INT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; if (sym_matches_type(left, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -309,7 +309,7 @@ } case _GUARD_TOS_INT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -319,7 +319,7 @@ } case _BINARY_OP_MULTIPLY_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -328,7 +328,7 @@ } case _BINARY_OP_ADD_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -337,7 +337,7 @@ } case _BINARY_OP_SUBTRACT_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -346,7 +346,7 @@ } case _GUARD_NOS_FLOAT: { - JitOptSymbol *left; + JitOptRef left; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -356,7 +356,7 @@ } case _GUARD_TOS_FLOAT: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -366,9 +366,9 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -390,14 +390,17 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } case _BINARY_OP_ADD_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -419,14 +422,17 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } case _BINARY_OP_SUBTRACT_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -448,14 +454,44 @@ res = sym_new_type(ctx, &PyFloat_Type); stack_pointer += -1; } + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); + } stack_pointer[-1] = res; break; } + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_OP_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -480,11 +516,11 @@ } case _BINARY_OP_INPLACE_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; + JitOptRef right; + JitOptRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - JitOptSymbol *res; + JitOptRef res; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); @@ -509,7 +545,7 @@ } case _BINARY_OP_EXTEND: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -518,8 +554,8 @@ } case _BINARY_SLICE: { - JitOptSymbol *container; - JitOptSymbol *res; + JitOptRef container; + JitOptRef res; container = stack_pointer[-3]; PyTypeObject *type = sym_get_type(container); if (type == &PyUnicode_Type || @@ -544,7 +580,7 @@ } case _BINARY_OP_SUBSCR_LIST_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -553,7 +589,7 @@ } case _BINARY_OP_SUBSCR_LIST_SLICE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -562,7 +598,7 @@ } case _BINARY_OP_SUBSCR_STR_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -571,7 +607,7 @@ } case _GUARD_NOS_TUPLE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -581,7 +617,7 @@ } case _GUARD_TOS_TUPLE: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -591,9 +627,9 @@ } case _BINARY_OP_SUBSCR_TUPLE_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *tuple_st; - JitOptSymbol *res; + JitOptRef sub_st; + JitOptRef tuple_st; + JitOptRef res; sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; assert(sym_matches_type(tuple_st, &PyTuple_Type)); @@ -620,7 +656,7 @@ } case _GUARD_NOS_DICT: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -630,7 +666,7 @@ } case _GUARD_TOS_DICT: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -640,7 +676,7 @@ } case _BINARY_OP_SUBSCR_DICT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -649,7 +685,7 @@ } case _BINARY_OP_SUBSCR_CHECK_FUNC: { - JitOptSymbol *getitem; + JitOptRef getitem; getitem = sym_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; @@ -658,8 +694,8 @@ } case _BINARY_OP_SUBSCR_INIT_CALL: { - JitOptSymbol *new_frame; - new_frame = NULL; + JitOptRef new_frame; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3] = new_frame; stack_pointer += -2; @@ -704,14 +740,14 @@ } case _CALL_INTRINSIC_1: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -720,10 +756,10 @@ } case _RETURN_VALUE: { - JitOptSymbol *retval; - JitOptSymbol *res; + JitOptRef retval; + JitOptRef res; retval = stack_pointer[-1]; - JitOptSymbol *temp = retval; + JitOptRef temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -747,14 +783,14 @@ } case _GET_AITER: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - JitOptSymbol *awaitable; + JitOptRef awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -763,7 +799,7 @@ } case _GET_AWAITABLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -772,15 +808,15 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { - JitOptSymbol *gen_frame; - gen_frame = NULL; + JitOptRef gen_frame; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = gen_frame; break; } case _YIELD_VALUE: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_unknown(ctx); stack_pointer[-1] = value; break; @@ -793,7 +829,7 @@ } case _LOAD_COMMON_CONSTANT: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -802,7 +838,7 @@ } case _LOAD_BUILD_CLASS: { - JitOptSymbol *bc; + JitOptRef bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -821,8 +857,8 @@ } case _UNPACK_SEQUENCE: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[-1 + oparg]; (void)top; @@ -835,9 +871,9 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol *val1; - JitOptSymbol *val0; + JitOptRef seq; + JitOptRef val1; + JitOptRef val0; seq = stack_pointer[-1]; val0 = sym_tuple_getitem(ctx, seq, 0); val1 = sym_tuple_getitem(ctx, seq, 1); @@ -849,8 +885,8 @@ } case _UNPACK_SEQUENCE_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol **values; + JitOptRef seq; + JitOptRef *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { @@ -862,7 +898,7 @@ } case _UNPACK_SEQUENCE_LIST: { - JitOptSymbol **values; + JitOptRef *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -873,8 +909,8 @@ } case _UNPACK_EX: { - JitOptSymbol **values; - JitOptSymbol **top; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; (void)top; @@ -910,7 +946,7 @@ } case _LOAD_LOCALS: { - JitOptSymbol *locals; + JitOptRef locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -921,7 +957,7 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - JitOptSymbol *v; + JitOptRef v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -930,7 +966,7 @@ } case _LOAD_GLOBAL: { - JitOptSymbol **res; + JitOptRef *res; res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); stack_pointer += 1; @@ -939,7 +975,7 @@ } case _PUSH_NULL_CONDITIONAL: { - JitOptSymbol **null; + JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); @@ -958,7 +994,7 @@ } case _LOAD_GLOBAL_MODULE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -967,7 +1003,7 @@ } case _LOAD_GLOBAL_BUILTINS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -988,14 +1024,14 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -1014,7 +1050,7 @@ } case _BUILD_STRING: { - JitOptSymbol *str; + JitOptRef str; str = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -1023,7 +1059,7 @@ } case _BUILD_INTERPOLATION: { - JitOptSymbol *interpolation; + JitOptRef interpolation; interpolation = sym_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); @@ -1032,7 +1068,7 @@ } case _BUILD_TEMPLATE: { - JitOptSymbol *template; + JitOptRef template; template = sym_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; @@ -1041,8 +1077,8 @@ } case _BUILD_TUPLE: { - JitOptSymbol **values; - JitOptSymbol *tup; + JitOptRef *values; + JitOptRef tup; values = &stack_pointer[-oparg]; tup = sym_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; @@ -1052,7 +1088,7 @@ } case _BUILD_LIST: { - JitOptSymbol *list; + JitOptRef list; list = sym_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -1073,7 +1109,7 @@ } case _BUILD_SET: { - JitOptSymbol *set; + JitOptRef set; set = sym_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -1082,7 +1118,7 @@ } case _BUILD_MAP: { - JitOptSymbol *map; + JitOptRef map; map = sym_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -1113,7 +1149,7 @@ } case _LOAD_SUPER_ATTR_ATTR: { - JitOptSymbol *attr_st; + JitOptRef attr_st; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -1122,8 +1158,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - JitOptSymbol *attr; - JitOptSymbol *self_or_null; + JitOptRef attr; + JitOptRef self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -1134,9 +1170,9 @@ } case _LOAD_ATTR: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol **self_or_null; + JitOptRef owner; + JitOptRef attr; + JitOptRef *self_or_null; owner = stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; @@ -1151,7 +1187,7 @@ } case _GUARD_TYPE_VERSION: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); @@ -1178,7 +1214,7 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)offset; @@ -1187,14 +1223,14 @@ } case _LOAD_ATTR_MODULE: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand0; uint16_t index = (uint16_t)this_instr->operand0; (void)dict_version; (void)index; - attr = NULL; + attr = PyJitRef_NULL; if (sym_is_const(ctx, owner)) { PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { @@ -1209,7 +1245,7 @@ } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { attr = sym_new_not_null(ctx); } stack_pointer[-1] = attr; @@ -1217,7 +1253,7 @@ } case _LOAD_ATTR_WITH_HINT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)hint; @@ -1226,7 +1262,7 @@ } case _LOAD_ATTR_SLOT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)index; @@ -1235,7 +1271,7 @@ } case _CHECK_ATTR_CLASS: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); @@ -1251,8 +1287,8 @@ } case _LOAD_ATTR_CLASS: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1266,10 +1302,10 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - JitOptSymbol *new_frame; + JitOptRef new_frame; PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-1] = new_frame; break; @@ -1300,7 +1336,7 @@ } case _COMPARE_OP: { - JitOptSymbol *res; + JitOptRef res; if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } @@ -1314,7 +1350,7 @@ } case _COMPARE_OP_FLOAT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1323,9 +1359,9 @@ } case _COMPARE_OP_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { @@ -1355,7 +1391,7 @@ } case _COMPARE_OP_STR: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; @@ -1364,7 +1400,7 @@ } case _IS_OP: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1373,7 +1409,7 @@ } case _CONTAINS_OP: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1382,7 +1418,7 @@ } case _GUARD_TOS_ANY_SET: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) @@ -1393,7 +1429,7 @@ } case _CONTAINS_OP_SET: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1402,7 +1438,7 @@ } case _CONTAINS_OP_DICT: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -1411,8 +1447,8 @@ } case _CHECK_EG_MATCH: { - JitOptSymbol *rest; - JitOptSymbol *match; + JitOptRef rest; + JitOptRef match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1421,14 +1457,14 @@ } case _CHECK_EXC_MATCH: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1437,7 +1473,7 @@ } case _IMPORT_FROM: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1450,15 +1486,15 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - JitOptSymbol *obj; - JitOptSymbol *len; + JitOptRef obj; + JitOptRef len; obj = stack_pointer[-1]; int tuple_length = sym_tuple_length(obj); if (tuple_length == -1) { @@ -1487,7 +1523,7 @@ } case _MATCH_CLASS: { - JitOptSymbol *attrs; + JitOptRef attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1496,7 +1532,7 @@ } case _MATCH_MAPPING: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1505,7 +1541,7 @@ } case _MATCH_SEQUENCE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1514,7 +1550,7 @@ } case _MATCH_KEYS: { - JitOptSymbol *values_or_none; + JitOptRef values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1523,9 +1559,9 @@ } case _GET_ITER: { - JitOptSymbol *iterable; - JitOptSymbol *iter; - JitOptSymbol *index_or_null; + JitOptRef iterable; + JitOptRef iter; + JitOptRef index_or_null; iterable = stack_pointer[-1]; if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { iter = iterable; @@ -1543,7 +1579,7 @@ } case _GET_YIELD_FROM_ITER: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1552,7 +1588,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1575,7 +1611,7 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ case _ITER_NEXT_LIST_TIER_TWO: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1584,7 +1620,7 @@ } case _ITER_CHECK_TUPLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = stack_pointer[-2]; if (sym_matches_type(iter, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1600,7 +1636,7 @@ } case _ITER_NEXT_TUPLE: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1619,7 +1655,7 @@ } case _ITER_NEXT_RANGE: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; @@ -1628,8 +1664,8 @@ } case _FOR_ITER_GEN_FRAME: { - JitOptSymbol *gen_frame; - gen_frame = NULL; + JitOptRef gen_frame; + gen_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[0] = gen_frame; stack_pointer += 1; @@ -1638,8 +1674,8 @@ } case _INSERT_NULL: { - JitOptSymbol *self; - JitOptSymbol **method_and_self; + JitOptRef self; + JitOptRef *method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; method_and_self[0] = sym_new_null(ctx); @@ -1650,7 +1686,7 @@ } case _LOAD_SPECIAL: { - JitOptSymbol **method_and_self; + JitOptRef *method_and_self; method_and_self = &stack_pointer[-2]; method_and_self[0] = sym_new_not_null(ctx); method_and_self[1] = sym_new_unknown(ctx); @@ -1658,7 +1694,7 @@ } case _WITH_EXCEPT_START: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1667,8 +1703,8 @@ } case _PUSH_EXC_INFO: { - JitOptSymbol *prev_exc; - JitOptSymbol *new_exc; + JitOptRef prev_exc; + JitOptRef new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1687,9 +1723,9 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1707,9 +1743,9 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1727,8 +1763,8 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1742,8 +1778,8 @@ } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1761,9 +1797,9 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1781,9 +1817,9 @@ } case _MAYBE_EXPAND_METHOD: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1800,7 +1836,7 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - JitOptSymbol *new_frame; + JitOptRef new_frame; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); co = get_code_with_logging((this_instr + 2)); @@ -1808,7 +1844,7 @@ ctx->done = true; break; } - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1816,7 +1852,7 @@ } case _CHECK_FUNCTION_VERSION: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { @@ -1833,7 +1869,7 @@ } case _CHECK_METHOD_VERSION: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { @@ -1855,7 +1891,7 @@ } case _CALL_NON_PY_GENERAL: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1864,8 +1900,8 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *null; - JitOptSymbol *callable; + JitOptRef null; + JitOptRef callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_null(null); @@ -1874,8 +1910,8 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; callable = sym_new_not_null(ctx); @@ -1893,8 +1929,8 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(sym_matches_type(callable, &PyFunction_Type)); @@ -1921,9 +1957,9 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *new_frame; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int argcount = oparg; @@ -1934,16 +1970,16 @@ ctx->done = true; break; } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { args--; argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = (JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; @@ -1952,12 +1988,12 @@ } case _PUSH_FRAME: { - JitOptSymbol *new_frame; + JitOptRef new_frame; new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = (_Py_UOpsAbstractFrame *)new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; co = get_code(this_instr); @@ -1984,7 +2020,7 @@ } case _GUARD_NOS_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-2]; if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1994,7 +2030,7 @@ } case _GUARD_NOS_NOT_NULL: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_is_not_null(nos)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2004,7 +2040,7 @@ } case _GUARD_THIRD_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-3]; if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2014,7 +2050,7 @@ } case _GUARD_CALLABLE_TYPE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2024,8 +2060,8 @@ } case _CALL_TYPE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; PyObject* type = (PyObject *)sym_get_type(arg); if (type) { @@ -2043,7 +2079,7 @@ } case _GUARD_CALLABLE_STR_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2053,8 +2089,8 @@ } case _CALL_STR_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyUnicode_Type)) { res = arg; @@ -2069,7 +2105,7 @@ } case _GUARD_CALLABLE_TUPLE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -2079,8 +2115,8 @@ } case _CALL_TUPLE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyTuple_Type)) { res = arg; @@ -2095,9 +2131,9 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -2112,8 +2148,8 @@ } case _CREATE_INIT_FRAME: { - JitOptSymbol *init_frame; - init_frame = NULL; + JitOptRef init_frame; + init_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; @@ -2128,7 +2164,7 @@ } case _CALL_BUILTIN_CLASS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2137,7 +2173,7 @@ } case _CALL_BUILTIN_O: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2146,7 +2182,7 @@ } case _CALL_BUILTIN_FAST: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2155,7 +2191,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2164,7 +2200,7 @@ } case _GUARD_CALLABLE_LEN: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { @@ -2175,7 +2211,7 @@ } case _CALL_LEN: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyLong_Type); stack_pointer[-3] = res; stack_pointer += -2; @@ -2184,7 +2220,7 @@ } case _GUARD_CALLABLE_ISINSTANCE: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { @@ -2195,9 +2231,9 @@ } case _CALL_ISINSTANCE: { - JitOptSymbol *cls; - JitOptSymbol *instance; - JitOptSymbol *res; + JitOptRef cls; + JitOptRef instance; + JitOptRef res; cls = stack_pointer[-1]; instance = stack_pointer[-2]; res = sym_new_type(ctx, &PyBool_Type); @@ -2218,7 +2254,7 @@ } case _GUARD_CALLABLE_LIST_APPEND: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { @@ -2235,7 +2271,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2244,7 +2280,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2253,7 +2289,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2262,7 +2298,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2279,8 +2315,8 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - JitOptSymbol *new_frame; - new_frame = NULL; + JitOptRef new_frame; + new_frame = PyJitRef_NULL; ctx->done = true; stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; @@ -2305,7 +2341,7 @@ } case _CALL_KW_NON_PY: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -2320,14 +2356,14 @@ /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - JitOptSymbol *func; + JitOptRef func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - JitOptSymbol *func_out; + JitOptRef func_out; func_out = sym_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; @@ -2336,7 +2372,7 @@ } case _RETURN_GENERATOR: { - JitOptSymbol *res; + JitOptRef res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -2358,7 +2394,7 @@ } case _BUILD_SLICE: { - JitOptSymbol *slice; + JitOptRef slice; slice = sym_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; @@ -2367,21 +2403,21 @@ } case _CONVERT_VALUE: { - JitOptSymbol *result; + JitOptRef result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -2390,8 +2426,8 @@ } case _COPY: { - JitOptSymbol *bottom; - JitOptSymbol *top; + JitOptRef bottom; + JitOptRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2402,9 +2438,9 @@ } case _BINARY_OP: { - JitOptSymbol *rhs; - JitOptSymbol *lhs; - JitOptSymbol *res; + JitOptRef rhs; + JitOptRef lhs; + JitOptRef res; rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; bool lhs_int = sym_matches_type(lhs, &PyLong_Type); @@ -2447,11 +2483,11 @@ } case _SWAP: { - JitOptSymbol *top; - JitOptSymbol *bottom; + JitOptRef top; + JitOptRef bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -2479,7 +2515,7 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -2493,7 +2529,7 @@ } case _GUARD_IS_FALSE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -2507,7 +2543,7 @@ } case _GUARD_IS_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); @@ -2525,7 +2561,7 @@ } case _GUARD_IS_NOT_NONE_POP: { - JitOptSymbol *val; + JitOptRef val; val = stack_pointer[-1]; if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); @@ -2573,9 +2609,9 @@ } case _LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2583,17 +2619,17 @@ } case _POP_TOP_LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2619,15 +2655,15 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _POP_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -2636,9 +2672,9 @@ } case _POP_CALL_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2646,9 +2682,9 @@ } case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2656,9 +2692,9 @@ } case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -2666,8 +2702,8 @@ } case _LOAD_CONST_UNDER_INLINE: { - JitOptSymbol *value; - JitOptSymbol *new; + JitOptRef value; + JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); stack_pointer[-1] = value; @@ -2678,8 +2714,8 @@ } case _LOAD_CONST_UNDER_INLINE_BORROW: { - JitOptSymbol *value; - JitOptSymbol *new; + JitOptRef value; + JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); stack_pointer[-1] = value; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 25de5d83166f64..838f25ad288ab0 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -88,6 +88,12 @@ out_of_space(JitOptContext *ctx) return &NO_SPACE_SYMBOL; } +JitOptRef +out_of_space_ref(JitOptContext *ctx) +{ + return PyJitRef_Wrap(out_of_space(ctx)); +} + static JitOptSymbol * sym_new(JitOptContext *ctx) { @@ -98,7 +104,7 @@ sym_new(JitOptContext *ctx) return NULL; } ctx->t_arena.ty_curr_number++; - self->tag = JIT_SYM_UNKNOWN_TAG; + self->tag = JIT_SYM_UNKNOWN_TAG;; return self; } @@ -117,25 +123,28 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_bottom(JitOptSymbol *sym) +_Py_uop_sym_is_bottom(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_not_null(JitOptSymbol *sym) { +_Py_uop_sym_is_not_null(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return true; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return false; } @@ -146,21 +155,22 @@ _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_null(JitOptSymbol *sym) +_Py_uop_sym_is_null(JitOptRef ref) { - return sym->tag == JIT_SYM_NULL_TAG; + return PyJitRef_Unwrap(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * -_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return sym->value.value; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return NULL; } @@ -172,8 +182,9 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -222,11 +233,12 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) } bool -_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version) +_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); PyTypeObject *type = _PyType_LookupByVersion(version); if (type) { - _Py_uop_sym_set_type(ctx, sym, type); + _Py_uop_sym_set_type(ctx, ref, type); } JitSymType tag = sym->tag; switch(tag) { @@ -279,8 +291,9 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int } void -_Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val) +_Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -301,10 +314,10 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TUPLE_TAG: if (PyTuple_CheckExact(const_val)) { - Py_ssize_t len = _Py_uop_sym_tuple_length(sym); + Py_ssize_t len = _Py_uop_sym_tuple_length(ref); if (len == PyTuple_GET_SIZE(const_val)) { for (Py_ssize_t i = 0; i < len; i++) { - JitOptSymbol *sym_item = _Py_uop_sym_tuple_getitem(ctx, sym, i); + JitOptRef sym_item = _Py_uop_sym_tuple_getitem(ctx, ref, i); PyObject *item = PyTuple_GET_ITEM(const_val, i); _Py_uop_sym_set_const(ctx, sym_item, item); } @@ -329,13 +342,14 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TRUTHINESS_TAG: if (!PyBool_Check(const_val) || - (_Py_uop_sym_is_const(ctx, sym) && - _Py_uop_sym_get_const(ctx, sym) != const_val)) + (_Py_uop_sym_is_const(ctx, ref) && + _Py_uop_sym_get_const(ctx, ref) != const_val)) { sym_set_bottom(ctx, sym); return; } - JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; + JitOptRef value = PyJitRef_Wrap( + allocation_base(ctx) + sym->truthiness.value); PyTypeObject *type = _Py_uop_sym_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: @@ -360,8 +374,9 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val } void -_Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NULL_TAG; } @@ -371,8 +386,9 @@ _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NON_NULL_TAG; } @@ -381,66 +397,69 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } } - -JitOptSymbol * +JitOptRef _Py_uop_sym_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } res->tag = JIT_SYM_NON_NULL_TAG; - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_type(ctx, res, typ); - return res; + JitOptRef ref = PyJitRef_Wrap(res); + _Py_uop_sym_set_type(ctx, ref, typ); + return ref; } // Adds a new reference to const_val, owned by the symbol. -JitOptSymbol * +JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_const(ctx, res, const_val); - return res; + JitOptRef ref = PyJitRef_Wrap(res); + _Py_uop_sym_set_const(ctx, ref, const_val); + return ref; } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); if (null_sym == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_null(ctx, null_sym); - return null_sym; + JitOptRef ref = PyJitRef_Wrap(null_sym); + _Py_uop_sym_set_null(ctx, ref); + return ref; } PyTypeObject * -_Py_uop_sym_get_type(JitOptSymbol *sym) +_Py_uop_sym_get_type(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -463,8 +482,9 @@ _Py_uop_sym_get_type(JitOptSymbol *sym) } unsigned int -_Py_uop_sym_get_type_version(JitOptSymbol *sym) +_Py_uop_sym_get_type_version(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -487,27 +507,28 @@ _Py_uop_sym_get_type_version(JitOptSymbol *sym) } bool -_Py_uop_sym_has_type(JitOptSymbol *sym) +_Py_uop_sym_has_type(JitOptRef sym) { return _Py_uop_sym_get_type(sym) != NULL; } bool -_Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); return _Py_uop_sym_get_type(sym) == typ; } bool -_Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version) +_Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version) { return _Py_uop_sym_get_type_version(sym) == version; } int -_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); switch(sym->tag) { case JIT_SYM_NULL_TAG: case JIT_SYM_TYPE_VERSION_TAG: @@ -527,7 +548,8 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) case JIT_SYM_TRUTHINESS_TAG: ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, + PyJitRef_Wrap(value)); if (truthiness < 0) { return truthiness; } @@ -553,12 +575,12 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) return -1; } -JitOptSymbol * -_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) +JitOptRef +_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } if (size > MAX_SYMBOLIC_TUPLE_SIZE) { res->tag = JIT_SYM_KNOWN_CLASS_TAG; @@ -568,15 +590,16 @@ _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) res->tag = JIT_SYM_TUPLE_TAG; res->tuple.length = size; for (int i = 0; i < size; i++) { - res->tuple.items[i] = (uint16_t)(args[i] - allocation_base(ctx)); + res->tuple.items[i] = (uint16_t)(PyJitRef_Unwrap(args[i]) - allocation_base(ctx)); } } - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * -_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) +JitOptRef +_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, int item) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; @@ -585,14 +608,15 @@ _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { - return allocation_base(ctx) + sym->tuple.items[item]; + return PyJitRef_Wrap(allocation_base(ctx) + sym->tuple.items[item]); } return _Py_uop_sym_new_not_null(ctx); } int -_Py_uop_sym_tuple_length(JitOptSymbol *sym) +_Py_uop_sym_tuple_length(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple)) { @@ -607,7 +631,7 @@ _Py_uop_sym_tuple_length(JitOptSymbol *sym) // Return true if known to be immortal. bool -_Py_uop_sym_is_immortal(JitOptSymbol *sym) +_Py_uop_symbol_is_immortal(JitOptSymbol *sym) { if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return _Py_IsImmortal(sym->value.value); @@ -621,19 +645,27 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) return false; } -JitOptSymbol * -_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) +bool +_Py_uop_sym_is_immortal(JitOptRef ref) +{ + JitOptSymbol *sym = PyJitRef_Unwrap(ref); + return _Py_uop_symbol_is_immortal(sym); +} + +JitOptRef +_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) { + JitOptSymbol *value = PyJitRef_Unwrap(ref); // It's clearer to invert this in the signature: bool invert = !truthy; if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.invert == invert) { - return value; + return ref; } JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, ref); if (truthiness < 0) { res->tag = JIT_SYM_TRUTHINESS_TAG; res->truthiness.invert = invert; @@ -642,7 +674,7 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) else { make_const(res, (truthiness ^ invert) ? Py_True : Py_False); } - return res; + return PyJitRef_Wrap(res); } // 0 on success, -1 on error. @@ -651,7 +683,7 @@ _Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); @@ -676,14 +708,14 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptSymbol *local = _Py_uop_sym_new_unknown(ctx); + JitOptRef local = _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - JitOptSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + JitOptRef stackvar = _Py_uop_sym_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -709,12 +741,12 @@ _Py_uop_abstractcontext_fini(JitOptContext *ctx) void _Py_uop_abstractcontext_init(JitOptContext *ctx) { - static_assert(sizeof(JitOptSymbol) <= 2 * sizeof(uint64_t), "JitOptSymbol has grown"); + static_assert(sizeof(JitOptSymbol) <= 3 * sizeof(uint64_t), "JitOptSymbol has grown"); ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; + ctx->locals_and_stack[i] = PyJitRef_NULL; } #endif @@ -767,44 +799,44 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - JitOptSymbol *sym = _Py_uop_sym_new_unknown(ctx); - if (sym == NULL) { + JitOptRef ref = _Py_uop_sym_new_unknown(ctx); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "top is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "top as constant is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(ref), "top is bottom"); - sym = make_bottom(ctx); - if (sym == NULL) { + ref = PyJitRef_Wrap(make_bottom(ctx)); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "bottom as constant is not NULL"); - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "bottom isn't bottom"); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "int is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "int as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "int as constant is not NULL"); - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); assert(val_42 != NULL); @@ -814,84 +846,84 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(val_43 != NULL); assert(_Py_IsImmortal(val_43)); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "42 as constant isn't 42"); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "42 is not immortal"); - - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "(42 and 42) as constant isn't 42"); - - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); - - sym = _Py_uop_sym_new_type(ctx, &PyBool_Type); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "a bool is not immortal"); - - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + _Py_uop_sym_set_const(ctx, ref, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "42 is not immortal"); + + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and float) isn't bottom"); + + ref = _Py_uop_sym_new_type(ctx, &PyBool_Type); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "a bool is not immortal"); + + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + _Py_uop_sym_set_const(ctx, ref, val_42); + _Py_uop_sym_set_const(ctx, ref, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and 43) isn't bottom"); - sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); - sym = _Py_uop_sym_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); - sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(0) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(None) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(False) is not False"); + ref = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(0) is not False"); - JitOptSymbol *i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); - JitOptSymbol *i2 = _Py_uop_sym_new_const(ctx, val_43); - JitOptSymbol *array[2] = { i1, i2 }; - sym = _Py_uop_sym_new_tuple(ctx, 2, array); + JitOptRef i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); + JitOptRef i2 = _Py_uop_sym_new_const(ctx, val_43); + JitOptRef array[2] = { i1, i2 }; + ref = _Py_uop_sym_new_tuple(ctx, 2, array); TEST_PREDICATE( - _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, sym, 0), &PyFloat_Type), + _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, ref, 0), &PyFloat_Type), "tuple item does not match value used to create tuple" ); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; tuple = _PyTuple_FromArray(pair, 2); - sym = _Py_uop_sym_new_const(ctx, tuple); + ref = _Py_uop_sym_new_const(ctx, tuple); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); - sym = _Py_uop_sym_new_type(ctx, &PyTuple_Type); + ref = _Py_uop_sym_new_type(ctx, &PyTuple_Type); TEST_PREDICATE( - _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, sym, 42)), + _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, ref, 42)), "Unknown tuple item is not narrowed to non-NULL" ); - JitOptSymbol *value = _Py_uop_sym_new_type(ctx, &PyBool_Type); - sym = _Py_uop_sym_new_truthiness(ctx, value, false); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == -1, "truthiness is not unknown"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == false, "truthiness is constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "truthiness is not NULL"); + JitOptRef value = _Py_uop_sym_new_type(ctx, &PyBool_Type); + ref = _Py_uop_sym_new_truthiness(ctx, value, false); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == -1, "truthiness is not unknown"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == false, "truthiness is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "truthiness is not NULL"); TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == false, "value is constant"); TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == NULL, "value is not NULL"); - _Py_uop_sym_set_const(ctx, sym, Py_False); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "truthiness is not True"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == true, "truthiness is not constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == Py_False, "truthiness is not False"); + _Py_uop_sym_set_const(ctx, ref, Py_False); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "truthiness is not True"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == true, "truthiness is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == Py_False, "truthiness is not False"); TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); _Py_uop_abstractcontext_fini(ctx); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c6a9fbcad8891f..2723e7ed7242ea 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -741,7 +741,7 @@ def visit(stmt: Stmt) -> None: continue #if not tkn.text.startswith(("Py", "_Py", "monitor")): # continue - if tkn.text.startswith(("sym_", "optimize_")): + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): # Optimize functions continue if tkn.text.endswith("Check"): diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 75805dbd7f37f4..3b4fe64b02a807 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -72,8 +72,8 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return "JitOptSymbol **" - return "JitOptSymbol *" + return "JitOptRef *" + return "JitOptRef " def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy