Skip to content

Commit abcca8a

Browse files
ericsnowcurrentlyfsc-eriker
ericsnowcurrently
authored and
fsc-eriker
committed
pythongh-76785: Improved Subinterpreters Compatibility with 3.12 (pythongh-115424)
For the most part, these changes make is substantially easier to backport subinterpreter-related code to 3.12, especially the related modules (e.g. _xxsubinterpreters). The main motivation is to support releasing a PyPI package with the 3.13 capabilities compiled for 3.12. A lot of the changes here involve either hiding details behind macros/functions or splitting up some files.
1 parent 67412c2 commit abcca8a

12 files changed

+857
-719
lines changed

‎Include/internal/pycore_code.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,15 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
12+
// We hide some of the newer PyCodeObject fields behind macros.
13+
// This helps with backporting certain changes to 3.12.
14+
#define _PyCode_HAS_EXECUTORS(CODE) \
15+
(CODE->co_executors != NULL)
16+
#define _PyCode_HAS_INSTRUMENTATION(CODE) \
17+
(CODE->_co_instrumentation_version > 0)
18+
19+
1120
#define CODE_MAX_WATCHERS 8
1221

1322
/* PEP 659

‎Include/internal/pycore_crossinterp.h

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,11 @@ struct _xid {
8787
PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void);
8888
PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data);
8989

90+
#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data)
91+
#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj)
92+
#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid)
93+
// Users should not need getters for "new_object" or "free".
94+
9095

9196
/* defining cross-interpreter data */
9297

@@ -101,6 +106,25 @@ PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize(
101106
PyAPI_FUNC(void) _PyCrossInterpreterData_Clear(
102107
PyInterpreterState *, _PyCrossInterpreterData *);
103108

109+
// Normally the Init* functions are sufficient. The only time
110+
// additional initialization might be needed is to set the "free" func,
111+
// though that should be infrequent.
112+
#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \
113+
do { \
114+
(DATA)->free = (FUNC); \
115+
} while (0)
116+
// Additionally, some shareable types are essentially light wrappers
117+
// around other shareable types. The crossinterpdatafunc of the wrapper
118+
// can often be implemented by calling the wrapped object's
119+
// crossinterpdatafunc and then changing the "new_object" function.
120+
// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that,
121+
// but might be better to have a function like
122+
// _PyCrossInterpreterData_AdaptToWrapper() instead.
123+
#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \
124+
do { \
125+
(DATA)->new_object = (FUNC); \
126+
} while (0)
127+
104128

105129
/* using cross-interpreter data */
106130

@@ -170,6 +194,8 @@ extern void _PyXI_Fini(PyInterpreterState *interp);
170194
extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp);
171195
extern void _PyXI_FiniTypes(PyInterpreterState *interp);
172196

197+
#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi)
198+
173199

174200
/***************************/
175201
/* short-term data sharing */

‎Include/internal/pycore_tstate.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,13 @@ extern "C" {
1313
#include "pycore_brc.h" // struct _brc_thread_state
1414

1515

16+
static inline void
17+
_PyThreadState_SetWhence(PyThreadState *tstate, int whence)
18+
{
19+
tstate->_whence = whence;
20+
}
21+
22+
1623
// Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
1724
// PyThreadState fields are exposed as part of the C API, although most fields
1825
// are intended to be private. The _PyThreadStateImpl fields not exposed.

‎Makefile.pre.in

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1671,6 +1671,14 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule
16711671

16721672
Modules/signalmodule.o: $(srcdir)/Modules/signalmodule.c $(srcdir)/Modules/posixmodule.h
16731673

1674+
Modules/_xxsubinterpretersmodule.o: $(srcdir)/Modules/_xxsubinterpretersmodule.c $(srcdir)/Modules/_interpreters_common.h
1675+
1676+
Modules/_xxinterpqueuesmodule.o: $(srcdir)/Modules/_xxinterpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h
1677+
1678+
Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h
1679+
1680+
Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h
1681+
16741682
Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile
16751683
$(CC) -c $(PY_CORE_CFLAGS) \
16761684
-DSOABI='"$(SOABI)"' \

‎Modules/_interpreters_common.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
2+
#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \
3+
PyInit_ ## NAME
4+
#define RESOLVE_MODINIT_FUNC_NAME(NAME) \
5+
_RESOLVE_MODINIT_FUNC_NAME(NAME)
6+
7+
8+
static int
9+
ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata)
10+
{
11+
//assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE);
12+
return _PyCrossInterpreterData_RegisterClass(cls, getdata);
13+
}

‎Modules/_xxinterpchannelsmodule.c

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
#include <sched.h> // sched_yield()
1818
#endif
1919

20+
#include "_interpreters_common.h"
21+
2022

2123
/*
2224
This module has the following process-global state:
@@ -80,7 +82,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*()
8082
API.. The module does not create any objects that are shared globally.
8183
*/
8284

83-
#define MODULE_NAME "_xxinterpchannels"
85+
#define MODULE_NAME _xxinterpchannels
86+
#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
87+
#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
8488

8589

8690
#define GLOBAL_MALLOC(TYPE) \
@@ -101,7 +105,7 @@ static int
101105
register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared,
102106
struct xid_class_registry *classes)
103107
{
104-
int res = _PyCrossInterpreterData_RegisterClass(cls, shared);
108+
int res = ensure_xid_class(cls, shared);
105109
if (res == 0) {
106110
assert(classes->count < MAX_XID_CLASSES);
107111
// The class has refs elsewhere, so we need to incref here.
@@ -167,7 +171,7 @@ _get_current_interp(void)
167171
static PyObject *
168172
_get_current_module(void)
169173
{
170-
PyObject *name = PyUnicode_FromString(MODULE_NAME);
174+
PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
171175
if (name == NULL) {
172176
return NULL;
173177
}
@@ -217,7 +221,7 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base)
217221
}
218222

219223
#define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \
220-
add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE)
224+
add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE)
221225

222226
static PyTypeObject *
223227
add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared,
@@ -299,7 +303,7 @@ _get_current_module_state(void)
299303
if (mod == NULL) {
300304
// XXX import it?
301305
PyErr_SetString(PyExc_RuntimeError,
302-
MODULE_NAME " module not imported yet");
306+
MODULE_NAME_STR " module not imported yet");
303307
return NULL;
304308
}
305309
module_state *state = get_module_state(mod);
@@ -784,7 +788,7 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid)
784788
while (next != NULL) {
785789
_channelitem *item = next;
786790
next = item->next;
787-
if (item->data->interpid == interpid) {
791+
if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
788792
if (prev == NULL) {
789793
queue->first = item->next;
790794
}
@@ -2126,7 +2130,7 @@ static PyStructSequence_Field channel_info_fields[] = {
21262130
};
21272131

21282132
static PyStructSequence_Desc channel_info_desc = {
2129-
.name = MODULE_NAME ".ChannelInfo",
2133+
.name = MODULE_NAME_STR ".ChannelInfo",
21302134
.doc = channel_info_doc,
21312135
.fields = channel_info_fields,
21322136
.n_in_sequence = 8,
@@ -2474,10 +2478,11 @@ struct _channelid_xid {
24742478
static PyObject *
24752479
_channelid_from_xid(_PyCrossInterpreterData *data)
24762480
{
2477-
struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
2481+
struct _channelid_xid *xid = \
2482+
(struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
24782483

24792484
// It might not be imported yet, so we can't use _get_current_module().
2480-
PyObject *mod = PyImport_ImportModule(MODULE_NAME);
2485+
PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR);
24812486
if (mod == NULL) {
24822487
return NULL;
24832488
}
@@ -2530,7 +2535,8 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj,
25302535
{
25312536
return -1;
25322537
}
2533-
struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
2538+
struct _channelid_xid *xid = \
2539+
(struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
25342540
xid->cid = ((channelid *)obj)->cid;
25352541
xid->end = ((channelid *)obj)->end;
25362542
xid->resolve = ((channelid *)obj)->resolve;
@@ -2601,7 +2607,7 @@ static PyType_Slot channelid_typeslots[] = {
26012607
};
26022608

26032609
static PyType_Spec channelid_typespec = {
2604-
.name = MODULE_NAME ".ChannelID",
2610+
.name = MODULE_NAME_STR ".ChannelID",
26052611
.basicsize = sizeof(channelid),
26062612
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
26072613
Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
@@ -2680,7 +2686,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj,
26802686
if (res < 0) {
26812687
return -1;
26822688
}
2683-
data->new_object = _channelend_from_xid;
2689+
_PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid);
26842690
return 0;
26852691
}
26862692

@@ -3379,7 +3385,7 @@ module_free(void *mod)
33793385

33803386
static struct PyModuleDef moduledef = {
33813387
.m_base = PyModuleDef_HEAD_INIT,
3382-
.m_name = MODULE_NAME,
3388+
.m_name = MODULE_NAME_STR,
33833389
.m_doc = module_doc,
33843390
.m_size = sizeof(module_state),
33853391
.m_methods = module_functions,
@@ -3390,7 +3396,7 @@ static struct PyModuleDef moduledef = {
33903396
};
33913397

33923398
PyMODINIT_FUNC
3393-
PyInit__xxinterpchannels(void)
3399+
MODINIT_FUNC_NAME(void)
33943400
{
33953401
return PyModuleDef_Init(&moduledef);
33963402
}

‎Modules/_xxinterpqueuesmodule.c

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,12 @@
88
#include "Python.h"
99
#include "pycore_crossinterp.h" // struct _xid
1010

11+
#include "_interpreters_common.h"
1112

12-
#define MODULE_NAME "_xxinterpqueues"
13+
14+
#define MODULE_NAME _xxinterpqueues
15+
#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
16+
#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
1317

1418

1519
#define GLOBAL_MALLOC(TYPE) \
@@ -64,7 +68,7 @@ _get_current_interp(void)
6468
static PyObject *
6569
_get_current_module(void)
6670
{
67-
PyObject *name = PyUnicode_FromString(MODULE_NAME);
71+
PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
6872
if (name == NULL) {
6973
return NULL;
7074
}
@@ -602,7 +606,7 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid)
602606
while (next != NULL) {
603607
_queueitem *item = next;
604608
next = item->next;
605-
if (item->data->interpid == interpid) {
609+
if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
606610
if (prev == NULL) {
607611
queue->items.first = item->next;
608612
}
@@ -1062,7 +1066,7 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type)
10621066
}
10631067
state->queue_type = (PyTypeObject *)Py_NewRef(queue_type);
10641068

1065-
if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) {
1069+
if (ensure_xid_class(queue_type, _queueobj_shared) < 0) {
10661070
return -1;
10671071
}
10681072

@@ -1130,7 +1134,7 @@ _queueid_xid_free(void *data)
11301134
static PyObject *
11311135
_queueobj_from_xid(_PyCrossInterpreterData *data)
11321136
{
1133-
int64_t qid = *(int64_t *)data->data;
1137+
int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data);
11341138
PyObject *qidobj = PyLong_FromLongLong(qid);
11351139
if (qidobj == NULL) {
11361140
return NULL;
@@ -1140,7 +1144,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data)
11401144
if (mod == NULL) {
11411145
// XXX import it?
11421146
PyErr_SetString(PyExc_RuntimeError,
1143-
MODULE_NAME " module not imported yet");
1147+
MODULE_NAME_STR " module not imported yet");
11441148
return NULL;
11451149
}
11461150

@@ -1181,7 +1185,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj,
11811185
_PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL,
11821186
_queueobj_from_xid);
11831187
Py_DECREF(qidobj);
1184-
data->free = _queueid_xid_free;
1188+
_PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free);
11851189
return 0;
11861190
}
11871191

@@ -1670,7 +1674,7 @@ module_free(void *mod)
16701674

16711675
static struct PyModuleDef moduledef = {
16721676
.m_base = PyModuleDef_HEAD_INIT,
1673-
.m_name = MODULE_NAME,
1677+
.m_name = MODULE_NAME_STR,
16741678
.m_doc = module_doc,
16751679
.m_size = sizeof(module_state),
16761680
.m_methods = module_functions,
@@ -1681,7 +1685,7 @@ static struct PyModuleDef moduledef = {
16811685
};
16821686

16831687
PyMODINIT_FUNC
1684-
PyInit__xxinterpqueues(void)
1688+
MODINIT_FUNC_NAME(void)
16851689
{
16861690
return PyModuleDef_Init(&moduledef);
16871691
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy