From bdd8d70ba1ff535a153a1f8ef2b2a08dad100116 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 11:47:51 -0600 Subject: [PATCH 01/28] InterpreterError inherits from Exception. --- Python/crossinterp_exceptions.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index 0f324bac48a2d8..6ecc10c7955fd8 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -6,9 +6,9 @@ static PyTypeObject _PyExc_InterpreterError = { .tp_name = "interpreters.InterpreterError", .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - //.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse, - //.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear, - //.tp_base = (PyTypeObject *)PyExc_BaseException, + //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, + //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear, + //.tp_base = (PyTypeObject *)PyExc_Exception, }; PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; @@ -19,8 +19,8 @@ static PyTypeObject _PyExc_InterpreterNotFoundError = { .tp_name = "interpreters.InterpreterNotFoundError", .tp_doc = PyDoc_STR("An interpreter was not found"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - //.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse, - //.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear, + //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, + //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear, .tp_base = &_PyExc_InterpreterError, }; PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; @@ -61,7 +61,7 @@ _get_not_shareable_error_type(PyInterpreterState *interp) static int init_exceptions(PyInterpreterState *interp) { - PyTypeObject *base = (PyTypeObject *)PyExc_BaseException; + PyTypeObject *base = (PyTypeObject *)PyExc_Exception; // builtin static types From 940f24df6cc19ec9fd0c8edd49f7709919529553 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Apr 2024 10:53:25 -0600 Subject: [PATCH 02/28] Add _PyInterpreterState_GetIDObject(). --- Include/internal/pycore_pystate.h | 3 +++ Modules/_interpreters_common.h | 17 ----------------- Modules/_xxinterpchannelsmodule.c | 5 ++--- Modules/_xxsubinterpretersmodule.c | 12 +++--------- Python/pystate.c | 14 ++++++++++++++ 5 files changed, 22 insertions(+), 29 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 35e266acd3ab60..eb5b5fee59009c 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -77,6 +77,9 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) interp == &_PyRuntime._main_interpreter); } +// Export for _xxsubinterpreters module. +PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); + // Export for _xxsubinterpreters module. PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index de9a60ce657e0c..07120f6ccc7207 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -19,20 +19,3 @@ clear_xid_class(PyTypeObject *cls) return _PyCrossInterpreterData_UnregisterClass(cls); } #endif - - -#ifdef RETURNS_INTERPID_OBJECT -static PyObject * -get_interpid_obj(PyInterpreterState *interp) -{ - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - assert(id < LLONG_MAX); - return PyLong_FromLongLong(id); -} -#endif diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index b63a3aab8263bc..bea0a6cf93fa02 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_crossinterp.h" // struct _xid #include "pycore_interp.h" // _PyInterpreterState_LookUpID() +#include "pycore_pystate.h" // _PyInterpreterState_GetIDObject() #ifdef MS_WINDOWS #define WIN32_LEAN_AND_MEAN @@ -17,9 +18,7 @@ #endif #define REGISTERS_HEAP_TYPES -#define RETURNS_INTERPID_OBJECT #include "_interpreters_common.h" -#undef RETURNS_INTERPID_OBJECT #undef REGISTERS_HEAP_TYPES @@ -2909,7 +2908,7 @@ channelsmod_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) goto except; } if (res) { - interpid_obj = get_interpid_obj(interp); + interpid_obj = _PyInterpreterState_GetIDObject(interp); if (interpid_obj == NULL) { goto except; } diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 94b8ee35001732..27ecd3fe216cea 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -20,9 +20,7 @@ #include "marshal.h" // PyMarshal_ReadObjectFromString() -#define RETURNS_INTERPID_OBJECT #include "_interpreters_common.h" -#undef RETURNS_INTERPID_OBJECT #define MODULE_NAME _xxsubinterpreters @@ -444,13 +442,9 @@ new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState assert(tstate != NULL); PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - if (_PyInterpreterState_IDInitref(interp) < 0) { - goto error; - } - if (p_idobj != NULL) { // We create the object using the original interpreter. - PyObject *idobj = get_interpid_obj(interp); + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); if (idobj == NULL) { goto error; } @@ -710,7 +704,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) interp = PyInterpreterState_Head(); while (interp != NULL) { - id = get_interpid_obj(interp); + id = _PyInterpreterState_GetIDObject(interp); if (id == NULL) { Py_DECREF(ids); return NULL; @@ -742,7 +736,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } - return get_interpid_obj(interp); + return _PyInterpreterState_GetIDObject(interp); } PyDoc_STRVAR(get_current_doc, diff --git a/Python/pystate.c b/Python/pystate.c index 892e740493cdfd..fecf6cb6ca506e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1167,6 +1167,20 @@ PyInterpreterState_GetID(PyInterpreterState *interp) return interp->id; } +PyObject * +_PyInterpreterState_GetIDObject(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IDInitref(interp) != 0) { + return NULL; + }; + int64_t interpid = interp->id; + if (interpid < 0) { + return NULL; + } + assert(interpid < LLONG_MAX); + return PyLong_FromLongLong(interpid); +} + int _PyInterpreterState_IDInitref(PyInterpreterState *interp) From 528c6e328d110ab81144b953ebf963c09d23f71a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Apr 2024 11:58:27 -0600 Subject: [PATCH 03/28] Add _PyXI_NewInterpreter() and _PyXI_EndInterpreter(). --- Include/internal/pycore_crossinterp.h | 15 ++++ Modules/_testinternalcapi.c | 121 +++++++++++++++++++------- Modules/_xxsubinterpretersmodule.c | 65 ++------------ Python/crossinterp.c | 79 +++++++++++++++++ 4 files changed, 193 insertions(+), 87 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 63abef864ff87f..1d797f81d8b21a 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -313,6 +313,21 @@ PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); +/*************/ +/* other API */ +/*************/ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(PyInterpreterState *) _PyXI_NewInterpreter( + PyInterpreterConfig *config, + PyThreadState **p_tstate, + PyThreadState **p_save_tstate); +PyAPI_FUNC(void) _PyXI_EndInterpreter( + PyInterpreterState *interp, + PyThreadState *tstate, + PyThreadState **p_save_tstate); + + #ifdef __cplusplus } #endif diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c5d65a373906f2..366d5b5efa7e67 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1357,7 +1357,88 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -/* To run some code in a sub-interpreter. */ +static PyInterpreterState * +_new_interpreter(PyObject *configobj, + PyThreadState **p_tstate, PyThreadState **p_save_tstate) +{ + PyInterpreterConfig config; + if (configobj == NULL) { + config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return NULL; + } + int res = _PyInterpreterConfig_InitFromDict(&config, dict); + Py_DECREF(dict); + if (res < 0) { + return NULL; + } + } + + return _PyXI_NewInterpreter(&config, p_tstate, p_save_tstate); +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.create() +static PyObject * +create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"config", NULL}; + PyObject *configobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|O:create_interpreter", kwlist, + &configobj)) + { + return NULL; + } + + PyInterpreterState *interp = _new_interpreter(configobj, NULL, NULL); + if (interp == NULL) { + return NULL; + } + + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + _PyXI_EndInterpreter(interp, NULL, NULL); + return NULL; + } + + return idobj; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.destroy() +static PyObject * +destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O:destroy_interpreter", kwlist, + &idobj)) + { + return NULL; + } + + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + + _PyXI_EndInterpreter(interp, NULL, NULL); + Py_RETURN_NONE; +} + + +/* To run some code in a sub-interpreter. + +Generally you can use test.support.interpreters, +but we keep this helper as a distinct implementation. +That's especially important for testing test.support.interpreters. +*/ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -1371,42 +1452,18 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } - PyInterpreterConfig config; - PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); - if (dict == NULL) { - PyErr_Format(PyExc_TypeError, "bad config %R", configobj); - return NULL; - } - int res = _PyInterpreterConfig_InitFromDict(&config, dict); - Py_DECREF(dict); - if (res < 0) { + PyThreadState *save_tstate; + PyThreadState *substate; + PyInterpreterState *interp = _new_interpreter(configobj, &substate, &save_tstate); + if (interp == NULL) { return NULL; } - PyThreadState *mainstate = PyThreadState_Get(); - - PyThreadState_Swap(NULL); - - PyThreadState *substate; - PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - PyThreadState_Swap(mainstate); - _PyErr_SetFromPyStatus(status); - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed"); - _PyErr_ChainExceptions1(exc); - return NULL; - } - assert(substate != NULL); /* only initialise 'cflags.cf_flags' to test backwards compatibility */ PyCompilerFlags cflags = {0}; int r = PyRun_SimpleStringFlags(code, &cflags); - Py_EndInterpreter(substate); - PyThreadState_Swap(mainstate); + _PyXI_EndInterpreter(interp, substate, &save_tstate); return PyLong_FromLong(r); } @@ -1738,6 +1795,10 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"create_interpreter", _PyCFunction_CAST(create_interpreter), + METH_VARARGS | METH_KEYWORDS}, + {"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 27ecd3fe216cea..be571c8cd646ca 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -423,55 +423,6 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } -static PyInterpreterState * -new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate) -{ - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - PyThreadState *tstate = NULL; - // XXX Possible GILState issues? - PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); - return NULL; - } - assert(tstate != NULL); - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - - if (p_idobj != NULL) { - // We create the object using the original interpreter. - PyObject *idobj = _PyInterpreterState_GetIDObject(interp); - if (idobj == NULL) { - goto error; - } - *p_idobj = idobj; - } - - if (p_tstate != NULL) { - *p_tstate = tstate; - } - else { - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - } - - return interp; - -error: - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; -} - - static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -600,8 +551,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *idobj = NULL; - PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL); + PyInterpreterState *interp = _PyXI_NewInterpreter(&config, NULL, NULL); if (interp == NULL) { // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); @@ -611,6 +561,12 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + _PyXI_EndInterpreter(interp, NULL, NULL); + return NULL; + } + if (reqrefs) { // Decref to 0 will destroy the interpreter. _PyInterpreterState_RequireIDRef(interp, 1); @@ -672,12 +628,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } // Destroy the interpreter. - PyThreadState *tstate = PyThreadState_New(interp); - _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP); - // XXX Possible GILState issues? - PyThreadState *save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); + _PyXI_EndInterpreter(interp, NULL, NULL); Py_RETURN_NONE; } diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 16efe9c3958f87..5a07e45b60679d 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1682,3 +1682,82 @@ _PyXI_FiniTypes(PyInterpreterState *interp) { fini_exceptions(interp); } + + +/*************/ +/* other API */ +/*************/ + +PyInterpreterState * +_PyXI_NewInterpreter(PyInterpreterConfig *config, + PyThreadState **p_tstate, PyThreadState **p_save_tstate) +{ + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + assert(save_tstate != NULL); + + PyThreadState *tstate; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + if (PyStatus_Exception(status)) { + // Since no new thread state was created, there is no exception + // to propagate; raise a fresh one after swapping back in the + // old thread state. + PyThreadState_Swap(save_tstate); + _PyErr_SetFromPyStatus(status); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } + assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + if (p_tstate != NULL) { + // We leave the new thread state as the current one. + *p_tstate = tstate; + } + else { + // Throw away the initial tstate. + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + save_tstate = NULL; + } + if (p_save_tstate != NULL) { + *p_save_tstate = save_tstate; + } + return interp; +} + +void +_PyXI_EndInterpreter(PyInterpreterState *interp, + PyThreadState *tstate, PyThreadState **p_save_tstate) +{ + PyThreadState *cur_tstate = PyThreadState_GET(); + PyThreadState *save_tstate = NULL; + if (tstate == NULL) { + if (PyThreadState_GetInterpreter(cur_tstate) == interp) { + tstate = cur_tstate; + } + else { + tstate = PyThreadState_New(interp); + _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP); + assert(tstate != NULL); + save_tstate = PyThreadState_Swap(tstate); + } + } + else { + assert(PyThreadState_GetInterpreter(tstate) == interp); + if (tstate != cur_tstate) { + assert(PyThreadState_GetInterpreter(cur_tstate) != interp); + save_tstate = PyThreadState_Swap(tstate); + } + } + + Py_EndInterpreter(tstate); + + if (p_save_tstate != NULL) { + save_tstate = *p_save_tstate; + } + PyThreadState_Swap(save_tstate); +} From 0bd2e6bcb77929e1fd27433d84e3cf7b9c3f5d66 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 1 Apr 2024 12:04:13 -0600 Subject: [PATCH 04/28] Add _testinternalcapi.next_interpreter_id(). --- Modules/_testinternalcapi.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 366d5b5efa7e67..e1b30a86d83a66 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1479,6 +1479,13 @@ normalize_interp_id(PyObject *self, PyObject *idobj) return PyLong_FromLongLong(interpid); } +static PyObject * +next_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t interpid = _PyRuntime.interpreters.next_id; + return PyLong_FromLongLong(interpid); +} + static PyObject * unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -1803,6 +1810,7 @@ static PyMethodDef module_functions[] = { _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, + {"next_interpreter_id", next_interpreter_id, METH_NOARGS}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, From 76e32cdfe6b0353220989f2dea46da53a1ba7a2e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 10:30:22 -0600 Subject: [PATCH 05/28] Add _testinternalcapi.exec_interpreter(). --- Modules/_testinternalcapi.c | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e1b30a86d83a66..8df66713233560 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1432,6 +1432,59 @@ destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) Py_RETURN_NONE; } +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.destroy() +static PyObject * +exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", "code", "main", NULL}; + PyObject *idobj; + const char *code; + int runningmain = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "Os|$p:exec_interpreter", kwlist, + &idobj, &code, &runningmain)) + { + return NULL; + } + + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + + PyObject *res = NULL; + PyThreadState *tstate = PyThreadState_New(interp); + _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_EXEC); + + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + if (runningmain) { + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + goto finally; + } + } + + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + int r = PyRun_SimpleStringFlags(code, &cflags); + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } + + if (runningmain) { + _PyInterpreterState_SetNotRunningMain(interp); + } + + res = PyLong_FromLong(r); + +finally: + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + return res; +} + /* To run some code in a sub-interpreter. @@ -1806,6 +1859,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS}, {"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter), METH_VARARGS | METH_KEYWORDS}, + {"exec_interpreter", _PyCFunction_CAST(exec_interpreter), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, From 9b7bdc4d096a1d6b7e049f6dc6a772a24caf0aab Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 10:56:41 -0600 Subject: [PATCH 06/28] Sketch out tests. --- Lib/test/test_interpreters/test_api.py | 25 ++++- Lib/test/test_interpreters/utils.py | 143 +++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index a326b39fd234c7..a1a8dd07aa406d 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -157,6 +157,20 @@ def test_idempotent(self): id2 = id(interp) self.assertNotEqual(id1, id2) + def test_unmanaged(self): + with self.unmanaged_interpreter() as unmanaged: + print(unmanaged) + err = unmanaged.exec(dedent(f""" + import {interpreters.__name__} as interpreters + err = None + try: + interpreters.get_current() + except ValueError as exc: + err = exc + assert exc is not None + """)) + self.assertEqual(err, '') + class ListAllTests(TestBase): @@ -199,6 +213,9 @@ def test_idempotent(self): for interp1, interp2 in zip(actual, expected): self.assertIs(interp1, interp2) + def test_unmanaged(self): + ... + class InterpreterObjectTests(TestBase): @@ -1071,7 +1088,7 @@ def test_get_config(self): with self.subTest('main'): expected = _interpreters.new_config('legacy') expected.gil = 'own' - interpid = _interpreters.get_main() + interpid, _ = _interpreters.get_main() config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) @@ -1138,6 +1155,12 @@ def test_create(self): with self.assertRaises(ValueError): _interpreters.create(orig) + def test_current(self): + ... + + def test_list_all(self): + ... + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 5ade6762ea24ef..b52ad8ca258a58 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -1,6 +1,7 @@ import contextlib import os import os.path +import pickle import subprocess import sys import tempfile @@ -66,6 +67,111 @@ def run(): t.join() +class UnmanagedInterpreter: + + @classmethod + def start(cls, create_pipe, *, legacy=True): + r_in, _ = inpipe = create_pipe() + r_out, w_out = outpipe = create_pipe() + + script = dedent(f""" + import pickle, os, traceback, _testcapi + + # Send the interpreter ID. + interpid = _testcapi.get_current_interpid() + os.write({w_out}, pickle.dumps(interpid)) + + # Run exec requests until "done". + script = b'' + while True: + ch = os.read({r_in}, 1) + if ch == b'\0': + if not script: + # done! + break + + # Run the provided script. + try: + exec(script) + except Exception as exc: + traceback.print_exc() + err = traceback.format_exception_only(exc) + os.write(w_out, err.encode('utf-8')) + os.write(w_out, b'\0') + script = b'' + else: + script += ch + """) + def run(): + try: + if legacy: + rc = support.run_in_subinterp(script) + else: + rc = support.run_in_subinterp_with_config(script) + assert rc == 0, rc + except BaseException: + os.write(w_out, b'\0') + raise # re-raise + t = threading.Thread(target=run) + t.start() + try: + # Get the interpreter ID. + data = os.read(r_out, 10) + assert len(data) < 10, repr(data) + assert data != b'\0' + interpid = pickle.loads(data) + + return cls(interpid, t, inpipe, outpipe) + except BaseException: + os.write(w_out, b'\0') + raise # re-raise + + def __init__(self, id, thread, inpipe, outpipe): + self._id = id + self._thread = thread + self._r_in, self._w_in = inpipe + self._r_out, self._w_out = outpipe + + def __repr__(self): + return f'<{type(self).__name__} {self._id}>' + + def __enter__(self): + return self + + def __exit__(self, *args): + self.shutdown() + + @property + def id(self): + return self._id + + def exec(self, code): + if isinstance(code, str): + code = code.encode('utf-8') + elif not isinstance(code, bytes): + raise TypeError(f'expected str/bytes, got {code!r}') + if not code: + raise ValueError('empty code') + os.write(self._w_in, code) + os.write(self._w_in, b'\0') + + out = b'' + while True: + ch = os.read(self._r_out, 1) + if ch == b'\0': + break + out += ch + return out.decode('utf-8') + + def shutdown(self, *, wait=True): + t = self._thread + self._thread = None + if t is not None: + os.write(self._w_in, b'\0') + if wait: + t.join() + + class TestBase(unittest.TestCase): def tearDown(self): @@ -175,3 +281,40 @@ def assert_ns_equal(self, ns1, ns2, msg=None): diff = f'namespace({diff})' standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) + + def unmanaged_interpreter(self, *, legacy=True): + return UnmanagedInterpreter.start(self.pipe, legacy=legacy) +# @contextlib.contextmanager +# def unmanaged_interpreter(self, *, legacy=True): +# r_id, w_id = self.pipe() +# r_done, w_done = self.pipe() +# script = dedent(f""" +# import marshal, os, _testcapi +# +# # Send the interpreter ID. +# interpid = _testcapi.get_current_interpid() +# data = marshal.dumps(interpid) +# os.write({w_id}, data) +# +# # Wait for "done". +# os.read({r_done}, 1) +# """) +# rc = None +# def task(): +# nonlocal rc +# if legacy: +# rc = support.run_in_subinterp(script) +# else: +# rc = support.run_in_subinterp_with_config(script) +# t = threading.Thread(target=task) +# t.start() +# try: +# # Get the interpreter ID. +# data = os.read(r_id, 10) +# assert len(data) < 10, repr(data) +# yield marshal.loads(data) +# finally: +# # Send "done". +# os.write(w_done, b'\0') +# t.join() +# self.assertEqual(rc, 0) From fa28f9b4be380c2208e214039a7017d3348270f0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 27 Mar 2024 12:51:59 -0600 Subject: [PATCH 07/28] Flesh out the tests. --- Lib/test/test_interpreters/test_api.py | 109 +++++-- Lib/test/test_interpreters/utils.py | 383 ++++++++++++++++--------- 2 files changed, 345 insertions(+), 147 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index a1a8dd07aa406d..7a81e9cb04d442 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,6 +1,7 @@ import os import pickle -from textwrap import dedent +import sys +from textwrap import dedent, indent import threading import types import unittest @@ -11,7 +12,10 @@ _interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError -from .utils import _captured_script, _run_output, _running, TestBase +from .utils import ( + _captured_script, _run_output, _running, TestBase, + CapturingScript, +) class ModuleTests(TestBase): @@ -959,6 +963,23 @@ class LowLevelTests(TestBase): # encountered by the high-level module, thus they # mostly shouldn't matter as much. + def _run_string(self, interpid, script, maxout): + with CapturingScript(script, combined=False) as captured: + err = _interpreters.run_string(interpid, captured.script) + if err is not None: + return None, err + raw = captured.read(maxout) + return raw.decode('utf-8'), None + + def run_and_capture(self, interpid, script, maxout=1000): + text, err = self._run_string(interpid, script, maxout) + if err is not None: + print() + print(err.errdisplay, file=sys.stderr) + raise Exception(f'subinterpreter failed: {err.formatted}') + else: + return text + def test_new_config(self): # This test overlaps with # test.test_capi.test_misc.InterpreterConfigTests. @@ -1092,17 +1113,59 @@ def test_get_config(self): config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) - with self.subTest('isolated'): - expected = _interpreters.new_config('isolated') - interpid = _interpreters.create('isolated') - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) + def test_get_main(self): + interpid, owned = _interpreters.get_main() + self.assertEqual(interpid, 0) + self.assertFalse(owned) - with self.subTest('legacy'): - expected = _interpreters.new_config('legacy') - interpid = _interpreters.create('legacy') - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) + def test_get_current(self): + with self.subTest('main'): + main, _ = _interpreters.get_main() + interpid, owned = _interpreters.get_current() + self.assertEqual(interpid, main) + self.assertFalse(owned) + + with self.subTest('owned'): +# r, w = self.pipe() +# orig = _interpreters.create() +# _interpreters.run_string(orig, dedent(f""" +# import os +# import {_interpreters.__name__} as _interpreters +# interpid = _interpreters.get_current() +# os.write({w}, interpid.as_bytes(8, 'big')) +# """)) +# raw = os.read(r, 8) +# interpid = int.from_bytes(raw, 'big') +# self.assertEqual(interpid, orig) + orig = _interpreters.create() + text = self.run_and_capture(orig, f""" + import {_interpreters.__name__} as _interpreters + interpid, owned = _interpreters.get_current() + print(interpid) + print(owned) + """) + interpid, owned = text.split() + interpid = int(interpid) + owned = eval(owned) + self.assertEqual(interpid, orig) + self.assertTrue(owned) + + with self.subTest('external'): + err, text = self.run_external(f""" + import {_interpreters.__name__} as _interpreters + interpid, owned = _interpreters.get_current() + print(interpid) + print(owned) + """) + assert err is None, err + interpid, owned = text.split() + interpid = int(interpid) + owned = eval(owned) + self.assertEqual(interpid, orig) + self.assertTrue(owned) + + def test_list_all(self): + ... def test_create(self): isolated = _interpreters.new_config('isolated') @@ -1155,11 +1218,25 @@ def test_create(self): with self.assertRaises(ValueError): _interpreters.create(orig) - def test_current(self): - ... + def test_get_config(self): + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + interpid, _ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) - def test_list_all(self): - ... + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create('isolated') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create('legacy') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) if __name__ == '__main__': diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index b52ad8ca258a58..98e6bf721073be 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -2,10 +2,11 @@ import os import os.path import pickle +import select import subprocess import sys import tempfile -from textwrap import dedent +from textwrap import dedent, indent import threading import types import unittest @@ -16,6 +17,17 @@ from test.support import interpreters +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None +else: + run_in_interpreter = _testinternalcapi.run_in_subinterp_with_config + +def requires__testinternalcapi(func): + return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) + + def _captured_script(script): r, w = os.pipe() indented = script.replace('\n', '\n ') @@ -67,109 +79,243 @@ def run(): t.join() -class UnmanagedInterpreter: - - @classmethod - def start(cls, create_pipe, *, legacy=True): - r_in, _ = inpipe = create_pipe() - r_out, w_out = outpipe = create_pipe() - - script = dedent(f""" - import pickle, os, traceback, _testcapi - - # Send the interpreter ID. - interpid = _testcapi.get_current_interpid() - os.write({w_out}, pickle.dumps(interpid)) - - # Run exec requests until "done". - script = b'' - while True: - ch = os.read({r_in}, 1) - if ch == b'\0': - if not script: - # done! - break - - # Run the provided script. - try: - exec(script) - except Exception as exc: - traceback.print_exc() - err = traceback.format_exception_only(exc) - os.write(w_out, err.encode('utf-8')) - os.write(w_out, b'\0') - script = b'' - else: - script += ch - """) - def run(): - try: - if legacy: - rc = support.run_in_subinterp(script) - else: - rc = support.run_in_subinterp_with_config(script) - assert rc == 0, rc - except BaseException: - os.write(w_out, b'\0') +class CapturingScript: + """ + Embeds a script in a new script that captures stdout/stderr + and uses pipes to expose them. + """ + + WRAPPER = dedent(""" + import sys + w_out = {w_out} + w_err = {w_err} + stdout, stderr = orig = (sys.stdout, sys.stderr) + if w_out is not None: + if w_err == w_out: + stdout = stderr = open(w_out, 'w', encoding='utf-8') + elif w_err is not None: + stdout = open(w_out, 'w', encoding='utf-8') + stderr = open(w_err, 'w', encoding='utf-8') + else: + stdout = open(w_out, 'w', encoding='utf-8') + else: + assert w_err is not None + stderr = open(w_err, 'w', encoding='utf-8') + + sys.stdout = stdout + sys.stderr = stderr + try: + ######################### + # begin wrapped script + + {wrapped} + + # end wrapped script + ######################### + finally: + sys.stdout, sys.stderr = orig + """) + + def __init__(self, script, *, combined=True): + self._r_out, self._w_out = os.pipe() + if combined: + self._r_err = self._w_err = None + w_err = self._w_out + else: + self._r_err, self._w_err = os.pipe() + w_err = self._w_err + self._combined = combined + + self._script = self.WRAPPER.format( + w_out=self._w_out, + w_err=w_err, + wrapped=indent(script, ' '), + ) + + def __del__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @property + def script(self): + return self._script + + def close(self): + for fd in [self._r_out, self._w_out, self._r_err, self._w_err]: + if fd is not None: + try: + os.close(fd) + except OSError: + if exc.errno != 9: + raise # re-raise + # It was closed already. + self._r_out = self._w_out = self._r_err = self._w_err = None + + def read(self, n=None): + return self.read_stdout(n) + + def read_stdout(self, n=None): + try: + return os.read(self._r_out, n) + except OSError as exc: + if exc.errno != 9: raise # re-raise - t = threading.Thread(target=run) - t.start() + # It was closed already. + return b'' + + def read_stderr(self, n=None): + if self._combined: + return b'' + try: + return os.read(self._r_err, n) + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + return b'' + + +class WatchedScript: + """ + Embeds a script in a new script that identifies when the script finishes. + Captures any uncaught exception, and uses a pipe to expose it. + """ + + WRAPPER = dedent(""" + import os, traceback, json + w_done = {w_done} try: - # Get the interpreter ID. - data = os.read(r_out, 10) - assert len(data) < 10, repr(data) - assert data != b'\0' - interpid = pickle.loads(data) - - return cls(interpid, t, inpipe, outpipe) - except BaseException: - os.write(w_out, b'\0') - raise # re-raise - - def __init__(self, id, thread, inpipe, outpipe): - self._id = id - self._thread = thread - self._r_in, self._w_in = inpipe - self._r_out, self._w_out = outpipe - - def __repr__(self): - return f'<{type(self).__name__} {self._id}>' + os.write(w_done, b'\0') # started + except OSError: + # It was canceled. + pass + else: + try: + ######################### + # begin wrapped script + + {wrapped} + + # end wrapped script + ######################### + except Exception as exc: + text = json.dumps(dict( + type=dict( + __name__=type(exc).__name__, + __qualname__=type(exc).__qualname__, + __module__=type(exc).__module__, + ), + msg=str(exc), + formatted=traceback.format_exception_only(exc), + errdisplay=traceback.format_exception(exc), + )) + try: + os.write(w_done, text.encode('utf-8')) + except BrokenPipeError: + # It was closed already. + pass + except OSError: + if exc.errno != 9: + raise # re-raise + # It was closed already. + finally: + try: + os.close({w_done}) + except OSError: + if exc.errno != 9: + raise # re-raise + # It was closed already. + """) + + def __init__(self, script): + self._started = False + self._finished = False + self._r_done, self._w_done = os.pipe() + + wrapper = self.WRAPPER.format( + w_done=self._w_done, + wrapped=indent(script, ' '), + ) + t = threading.Thread(target=self._watch) + t.start() + + self._script = wrapper + self._thread = t + + def __del__(self): + self.close() def __enter__(self): return self def __exit__(self, *args): - self.shutdown() + self.close() @property - def id(self): - return self._id - - def exec(self, code): - if isinstance(code, str): - code = code.encode('utf-8') - elif not isinstance(code, bytes): - raise TypeError(f'expected str/bytes, got {code!r}') - if not code: - raise ValueError('empty code') - os.write(self._w_in, code) - os.write(self._w_in, b'\0') - - out = b'' - while True: - ch = os.read(self._r_out, 1) - if ch == b'\0': - break - out += ch - return out.decode('utf-8') - - def shutdown(self, *, wait=True): - t = self._thread - self._thread = None - if t is not None: - os.write(self._w_in, b'\0') - if wait: - t.join() + def script(self): + return self._script + + def close(self): + for fd in [self._r_done, self._w_done]: + if fd is not None: + try: + os.close(fd) + except OSError: + if exc.errno != 9: + raise # re-raise + # It was closed already. + self._r_done = None + self._w_done = None + + @property + def finished(self): + if isinstance(self._finished, dict): + exc = types.SimpleNamespace(**self._finished) + exc.type = types.SimpleNamespace(**exc.type) + return exc + return self._finished + + def _watch(self): + r_fd = self._r_done + + assert not self._started + try: + ch0 = os.read(r_fd, 1) + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + return + if ch0 == b'': + # The write end of the pipe has closed. + return + assert ch0 == b'\0', repr(ch0) + self._started = True + + assert not self._finished + data = b'' + try: + chunk = os.read(r_fd, 100) + self._finished = True + while chunk: + data += chunk + chunk = os.read(r_fd, 100) + os.close(r_fd) + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + + if data: + self._finished = json.loads(data) + + # It may be useful to implement the concurrent.futures.Future API + # on this class. class TestBase(unittest.TestCase): @@ -282,39 +428,14 @@ def assert_ns_equal(self, ns1, ns2, msg=None): standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) - def unmanaged_interpreter(self, *, legacy=True): - return UnmanagedInterpreter.start(self.pipe, legacy=legacy) -# @contextlib.contextmanager -# def unmanaged_interpreter(self, *, legacy=True): -# r_id, w_id = self.pipe() -# r_done, w_done = self.pipe() -# script = dedent(f""" -# import marshal, os, _testcapi -# -# # Send the interpreter ID. -# interpid = _testcapi.get_current_interpid() -# data = marshal.dumps(interpid) -# os.write({w_id}, data) -# -# # Wait for "done". -# os.read({r_done}, 1) -# """) -# rc = None -# def task(): -# nonlocal rc -# if legacy: -# rc = support.run_in_subinterp(script) -# else: -# rc = support.run_in_subinterp_with_config(script) -# t = threading.Thread(target=task) -# t.start() -# try: -# # Get the interpreter ID. -# data = os.read(r_id, 10) -# assert len(data) < 10, repr(data) -# yield marshal.loads(data) -# finally: -# # Send "done". -# os.write(w_done, b'\0') -# t.join() -# self.assertEqual(rc, 0) + @requires__testinternalcapi + def run_external(self, script, config='legacy'): + with CapturingScript(script, combined=False) as captured: + with WatchedScript(captured.script) as watched: + rc = run_in_interpreter(watched.script, config) + assert rc == 0, rc + text = watched.read(100).decode('utf-8') + err = captured.finished + if err is True: + err = None + return err, text From 5e31f6e35ff285ff599259f904fb3acd9aa2fd48 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 28 Mar 2024 10:51:49 -0600 Subject: [PATCH 08/28] Add PipeEnd. --- Lib/test/test_interpreters/test_api.py | 18 +- Lib/test/test_interpreters/utils.py | 237 +++++++++++++++++++++---- 2 files changed, 211 insertions(+), 44 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 7a81e9cb04d442..7d88c73becf27b 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -968,7 +968,8 @@ def _run_string(self, interpid, script, maxout): err = _interpreters.run_string(interpid, captured.script) if err is not None: return None, err - raw = captured.read(maxout) + raw = b'' + #raw = captured.read(maxout) return raw.decode('utf-8'), None def run_and_capture(self, interpid, script, maxout=1000): @@ -1126,17 +1127,6 @@ def test_get_current(self): self.assertFalse(owned) with self.subTest('owned'): -# r, w = self.pipe() -# orig = _interpreters.create() -# _interpreters.run_string(orig, dedent(f""" -# import os -# import {_interpreters.__name__} as _interpreters -# interpid = _interpreters.get_current() -# os.write({w}, interpid.as_bytes(8, 'big')) -# """)) -# raw = os.read(r, 8) -# interpid = int.from_bytes(raw, 'big') -# self.assertEqual(interpid, orig) orig = _interpreters.create() text = self.run_and_capture(orig, f""" import {_interpreters.__name__} as _interpreters @@ -1144,7 +1134,9 @@ def test_get_current(self): print(interpid) print(owned) """) - interpid, owned = text.split() + parts = text.split() + assert len(parts) == 2, parts + interpid, owned = parts interpid = int(interpid) owned = eval(owned) self.assertEqual(interpid, orig) diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 98e6bf721073be..c997de3007bcee 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -2,7 +2,8 @@ import os import os.path import pickle -import select +import queue +#import select import subprocess import sys import tempfile @@ -79,6 +80,156 @@ def run(): t.join() +@contextlib.contextmanager +def fd_maybe_closed(): + try: + yield + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # The file descriptor has closed. + + +class PipeEnd: + + def __init__(self, fd): + self._fd = fd + self._closed = False + self._lock = threading.Lock() + + def __del__(self): + self.close() + + def __str__(self): + return str(self._fd) + + def __repr__(self): + return f'{type_self.__name__}({self._fd!r})' + + def __index__(self): + return self._fd + + def close(self): + with self._lock: + if self._closed: + return + self._closed = True + with fd_maybe_closed(): + self._close() + + @contextlib.contextmanager + def _maybe_closed(self): + assert self._lock.locked() + with fd_maybe_closed(): + yield + return + # It was closed already. + if not self._closed: + self._closed = True + with fd_maybe_closed(): + self._close() + + def _close(self): + os.close(self._fd) + + +class ReadPipe(PipeEnd): + + def __init__(self, fd): + super().__init__(fd) + + self._requests = queue.Queue(1) + self._looplock = threading.Lock() + self._thread = threading.Thread(target=self._handle_read_requests) + self._thread.start() + self._buffer = bytearray() + self._bufferlock = threading.Lock() + + def _handle_read_requests(self): + while True: + try: + req = self._requests.get() + except queue.ShutDown: + # The pipe is closed. + break + finally: + # It was locked in either self.close() or self.read(). + assert self._lock.locked() + + # The loop lock was taken for us in self._read(). + try: + if req is None: + # Read all available bytes. + buf = bytearray() + data = os.read(self._fd, 8147) # a prime number + while data: + buf += data + data = os.read(self._fd, 8147) + with self._bufferlock: + self._buffer += buf + else: + assert req >= 0, repr(req) + with self._bufferlock: + missing = req - len(self._buffer) + if missing > 0: + with self._maybe_closed(): + data = os.read(self._fd, missing) + # The rest is skipped if its already closed. + with self._bufferlock: + self._buffer += data + finally: + self._looplock.release() + + def read(self, n=None, timeout=-1): + if n == 0: + return b'' + elif n is not None and n < 0: + # This will fail. + os.read(self._fd, n) + raise OSError('invalid argument') + + with self._lock: + # The looo will release it after handling the request. + self._looplock.acquire() + try: + self._requests.put_nowait(n) + except BaseException: + self._looplock.release() + raise # re-raise + # Wait for the request to finish. + if self._looplock.acquire(timeout=timeout): + self._looplock.release() + # Return (up to) the requested bytes. + with self._bufferlock: + data = self._buffer[:n] + self._buffer = self._buffer[n:] + return bytes(data) + + def _close(self): + # Ideally the write end is already closed. + self._requests.shutdown() + with self._bufferlock: + # Throw away any leftover bytes. + self._buffer = b'' + super()._close() + + +class WritePipe(PipeEnd): + + def write(self, n=None): + try: + with self._maybe_closed(): + return os.write(self._fd, n) + except BrokenPipeError: + # The read end was already closed. + self.close() + + +def create_pipe(): + r, w = os.pipe() + return ReadPipe(r), WritePipe(w) + + class CapturingScript: """ Embeds a script in a new script that captures stdout/stderr @@ -94,13 +245,13 @@ class CapturingScript: if w_err == w_out: stdout = stderr = open(w_out, 'w', encoding='utf-8') elif w_err is not None: - stdout = open(w_out, 'w', encoding='utf-8') - stderr = open(w_err, 'w', encoding='utf-8') + stdout = open(w_out, 'w', encoding='utf-8', closefd=False) + stderr = open(w_err, 'w', encoding='utf-8', closefd=False) else: - stdout = open(w_out, 'w', encoding='utf-8') + stdout = open(w_out, 'w', encoding='utf-8', closefd=False) else: assert w_err is not None - stderr = open(w_err, 'w', encoding='utf-8') + stderr = open(w_err, 'w', encoding='utf-8', closefd=False) sys.stdout = stdout sys.stderr = stderr @@ -117,12 +268,12 @@ class CapturingScript: """) def __init__(self, script, *, combined=True): - self._r_out, self._w_out = os.pipe() + self._r_out, self._w_out = create_pipe() if combined: self._r_err = self._w_err = None w_err = self._w_out else: - self._r_err, self._w_err = os.pipe() + self._r_err, self._w_err = create_pipe() w_err = self._w_err self._combined = combined @@ -132,6 +283,9 @@ def __init__(self, script, *, combined=True): wrapped=indent(script, ' '), ) + self._buf_stdout = None + self._buf_stderr = None + def __del__(self): self.close() @@ -146,38 +300,59 @@ def script(self): return self._script def close(self): - for fd in [self._r_out, self._w_out, self._r_err, self._w_err]: - if fd is not None: - try: - os.close(fd) - except OSError: - if exc.errno != 9: - raise # re-raise - # It was closed already. - self._r_out = self._w_out = self._r_err = self._w_err = None + if self._w_out is not None: + assert self._r_out is not None + self._w_out.close() + self._w_out = None + self._buf_stdout = self._r_out.read() + self._r_out.close() + self._r_out = None + else: + assert self._r_out is None + + if self._combined: + assert self._w_err is None + assert self._r_err is None + elif self._w_err is not None: + assert self._r_err is not None + self._w_err.close() + self._w_err = None + self._buf_stderr = self._r_err.read() + self._r_err.close() + self._r_err = None + else: + assert self._r_err is None def read(self, n=None): return self.read_stdout(n) def read_stdout(self, n=None): - try: - return os.read(self._r_out, n) - except OSError as exc: - if exc.errno != 9: - raise # re-raise - # It was closed already. - return b'' + if self._r_out is not None: + data = self._r_out.read(n) + elif self._buf_stdout is None: + data = b'' + elif n is None or n == len(self._buf_stdout): + data = self._buf_stdout + self._buf_stdout = None + else: + data = self._buf_stdout[:n] + self._buf_stdout = self._buf_stdout[n:] + return data def read_stderr(self, n=None): if self._combined: return b'' - try: - return os.read(self._r_err, n) - except OSError as exc: - if exc.errno != 9: - raise # re-raise - # It was closed already. - return b'' + if self._r_err is not None: + data = self._r_err.read(n) + elif self._buf_stderr is None: + data = b'' + elif n is None or n == len(self._buf_stderr): + data = self._buf_stderr + self._buf_stderr = None + else: + data = self._buf_stderr[:n] + self._buf_stderr = self._buf_stderr[n:] + return data class WatchedScript: @@ -435,7 +610,7 @@ def run_external(self, script, config='legacy'): rc = run_in_interpreter(watched.script, config) assert rc == 0, rc text = watched.read(100).decode('utf-8') - err = captured.finished + err = watched.finished if err is True: err = None return err, text From 9111a83fa1f4a46b9bb6e4716811e98f5e02de9a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 28 Mar 2024 15:31:06 -0600 Subject: [PATCH 09/28] Refactor _captured_script(). --- Lib/test/test_interpreters/test_api.py | 75 +-- Lib/test/test_interpreters/utils.py | 681 +++++++++---------------- 2 files changed, 282 insertions(+), 474 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 7d88c73becf27b..445a81211d6a4e 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -12,10 +12,7 @@ _interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError -from .utils import ( - _captured_script, _run_output, _running, TestBase, - CapturingScript, -) +from .utils import _captured_script, _run_output, _running, TestBase class ModuleTests(TestBase): @@ -542,10 +539,10 @@ class TestInterpreterExec(TestBase): def test_success(self): interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: + script, results = _captured_script('print("it worked!", end="")') + with results: interp.exec(script) - out = file.read() + out = results.stdout() self.assertEqual(out, 'it worked!') @@ -604,15 +601,15 @@ def script(): def test_in_thread(self): interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: + script, results = _captured_script('print("it worked!", end="")') + with results: def f(): interp.exec(script) t = threading.Thread(target=f) t.start() t.join() - out = file.read() + out = results.stdout() self.assertEqual(out, 'it worked!') @@ -963,24 +960,6 @@ class LowLevelTests(TestBase): # encountered by the high-level module, thus they # mostly shouldn't matter as much. - def _run_string(self, interpid, script, maxout): - with CapturingScript(script, combined=False) as captured: - err = _interpreters.run_string(interpid, captured.script) - if err is not None: - return None, err - raw = b'' - #raw = captured.read(maxout) - return raw.decode('utf-8'), None - - def run_and_capture(self, interpid, script, maxout=1000): - text, err = self._run_string(interpid, script, maxout) - if err is not None: - print() - print(err.errdisplay, file=sys.stderr) - raise Exception(f'subinterpreter failed: {err.formatted}') - else: - return text - def test_new_config(self): # This test overlaps with # test.test_capi.test_misc.InterpreterConfigTests. @@ -1126,35 +1105,37 @@ def test_get_current(self): self.assertEqual(interpid, main) self.assertFalse(owned) - with self.subTest('owned'): - orig = _interpreters.create() - text = self.run_and_capture(orig, f""" - import {_interpreters.__name__} as _interpreters - interpid, owned = _interpreters.get_current() - print(interpid) - print(owned) - """) + script = f""" + import {_interpreters.__name__} as _interpreters + interpid, owned = _interpreters.get_current() + print(interpid) + print(owned) + """ + def parse_stdout(text): parts = text.split() assert len(parts) == 2, parts interpid, owned = parts interpid = int(interpid) owned = eval(owned) + return interpid, owned + + with self.subTest('owned'): + orig = _interpreters.create() + text = self.run_and_capture(orig, script) + interpid, owned = parse_stdout(text) self.assertEqual(interpid, orig) self.assertTrue(owned) with self.subTest('external'): - err, text = self.run_external(f""" - import {_interpreters.__name__} as _interpreters - interpid, owned = _interpreters.get_current() - print(interpid) - print(owned) - """) + last = 0 + for id, *_ in _interpreters.list_all(): + last = max(last, id) + expected = last + 1 + err, text = self.run_external(script) assert err is None, err - interpid, owned = text.split() - interpid = int(interpid) - owned = eval(owned) - self.assertEqual(interpid, orig) - self.assertTrue(owned) + interpid, owned = parse_stdout(text) + self.assertEqual(interpid, expected) + self.assertFalse(owned) def test_list_all(self): ... diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index c997de3007bcee..7eb65baf681535 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -1,4 +1,6 @@ import contextlib +import json +import io import os import os.path import pickle @@ -11,10 +13,13 @@ import threading import types import unittest +import warnings from test import support from test.support import os_helper +from test.support import import_helper +_interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters @@ -29,356 +34,56 @@ def requires__testinternalcapi(func): return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) -def _captured_script(script): - r, w = os.pipe() - indented = script.replace('\n', '\n ') - wrapped = dedent(f""" - import contextlib - with open({w}, 'w', encoding='utf-8') as spipe: - with contextlib.redirect_stdout(spipe): - {indented} - """) - return wrapped, open(r, encoding='utf-8') - - -def clean_up_interpreters(): - for interp in interpreters.list_all(): - if interp.id == 0: # main - continue - try: - interp.close() - except RuntimeError: - pass # already destroyed - - -def _run_output(interp, request, init=None): - script, rpipe = _captured_script(request) - with rpipe: - if init: - interp.prepare_main(init) - interp.exec(script) - return rpipe.read() +def _dump_script(text): + lines = text.splitlines() + print('-' * 20) + for i, line in enumerate(lines, 1): + print(f' {i:>{len(str(len(lines)))}} {line}') + print('-' * 20) -@contextlib.contextmanager -def _running(interp): - r, w = os.pipe() - def run(): - interp.exec(dedent(f""" - # wait for "signal" - with open({r}) as rpipe: - rpipe.read() - """)) - - t = threading.Thread(target=run) - t.start() - - yield - - with open(w, 'w') as spipe: - spipe.write('done') - t.join() - - -@contextlib.contextmanager -def fd_maybe_closed(): +def _close_file(file): try: - yield + if hasattr(file, 'close'): + file.close() + else: + os.close(file) except OSError as exc: if exc.errno != 9: raise # re-raise - # The file descriptor has closed. - - -class PipeEnd: - - def __init__(self, fd): - self._fd = fd - self._closed = False - self._lock = threading.Lock() - - def __del__(self): - self.close() - - def __str__(self): - return str(self._fd) - - def __repr__(self): - return f'{type_self.__name__}({self._fd!r})' - - def __index__(self): - return self._fd - - def close(self): - with self._lock: - if self._closed: - return - self._closed = True - with fd_maybe_closed(): - self._close() - - @contextlib.contextmanager - def _maybe_closed(self): - assert self._lock.locked() - with fd_maybe_closed(): - yield - return # It was closed already. - if not self._closed: - self._closed = True - with fd_maybe_closed(): - self._close() - - def _close(self): - os.close(self._fd) -class ReadPipe(PipeEnd): - - def __init__(self, fd): - super().__init__(fd) - - self._requests = queue.Queue(1) - self._looplock = threading.Lock() - self._thread = threading.Thread(target=self._handle_read_requests) - self._thread.start() - self._buffer = bytearray() - self._bufferlock = threading.Lock() - - def _handle_read_requests(self): - while True: - try: - req = self._requests.get() - except queue.ShutDown: - # The pipe is closed. - break - finally: - # It was locked in either self.close() or self.read(). - assert self._lock.locked() - - # The loop lock was taken for us in self._read(). - try: - if req is None: - # Read all available bytes. - buf = bytearray() - data = os.read(self._fd, 8147) # a prime number - while data: - buf += data - data = os.read(self._fd, 8147) - with self._bufferlock: - self._buffer += buf - else: - assert req >= 0, repr(req) - with self._bufferlock: - missing = req - len(self._buffer) - if missing > 0: - with self._maybe_closed(): - data = os.read(self._fd, missing) - # The rest is skipped if its already closed. - with self._bufferlock: - self._buffer += data - finally: - self._looplock.release() - - def read(self, n=None, timeout=-1): - if n == 0: - return b'' - elif n is not None and n < 0: - # This will fail. - os.read(self._fd, n) - raise OSError('invalid argument') - - with self._lock: - # The looo will release it after handling the request. - self._looplock.acquire() - try: - self._requests.put_nowait(n) - except BaseException: - self._looplock.release() - raise # re-raise - # Wait for the request to finish. - if self._looplock.acquire(timeout=timeout): - self._looplock.release() - # Return (up to) the requested bytes. - with self._bufferlock: - data = self._buffer[:n] - self._buffer = self._buffer[n:] - return bytes(data) - - def _close(self): - # Ideally the write end is already closed. - self._requests.shutdown() - with self._bufferlock: - # Throw away any leftover bytes. - self._buffer = b'' - super()._close() - - -class WritePipe(PipeEnd): - - def write(self, n=None): - try: - with self._maybe_closed(): - return os.write(self._fd, n) - except BrokenPipeError: - # The read end was already closed. - self.close() - - -def create_pipe(): - r, w = os.pipe() - return ReadPipe(r), WritePipe(w) - - -class CapturingScript: - """ - Embeds a script in a new script that captures stdout/stderr - and uses pipes to expose them. - """ - - WRAPPER = dedent(""" - import sys - w_out = {w_out} - w_err = {w_err} - stdout, stderr = orig = (sys.stdout, sys.stderr) - if w_out is not None: - if w_err == w_out: - stdout = stderr = open(w_out, 'w', encoding='utf-8') - elif w_err is not None: - stdout = open(w_out, 'w', encoding='utf-8', closefd=False) - stderr = open(w_err, 'w', encoding='utf-8', closefd=False) - else: - stdout = open(w_out, 'w', encoding='utf-8', closefd=False) - else: - assert w_err is not None - stderr = open(w_err, 'w', encoding='utf-8', closefd=False) - - sys.stdout = stdout - sys.stderr = stderr - try: - ######################### - # begin wrapped script - - {wrapped} - - # end wrapped script - ######################### - finally: - sys.stdout, sys.stderr = orig - """) - - def __init__(self, script, *, combined=True): - self._r_out, self._w_out = create_pipe() - if combined: - self._r_err = self._w_err = None - w_err = self._w_out - else: - self._r_err, self._w_err = create_pipe() - w_err = self._w_err - self._combined = combined - - self._script = self.WRAPPER.format( - w_out=self._w_out, - w_err=w_err, - wrapped=indent(script, ' '), - ) - - self._buf_stdout = None - self._buf_stderr = None - - def __del__(self): - self.close() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - @property - def script(self): - return self._script - - def close(self): - if self._w_out is not None: - assert self._r_out is not None - self._w_out.close() - self._w_out = None - self._buf_stdout = self._r_out.read() - self._r_out.close() - self._r_out = None - else: - assert self._r_out is None - - if self._combined: - assert self._w_err is None - assert self._r_err is None - elif self._w_err is not None: - assert self._r_err is not None - self._w_err.close() - self._w_err = None - self._buf_stderr = self._r_err.read() - self._r_err.close() - self._r_err = None - else: - assert self._r_err is None - - def read(self, n=None): - return self.read_stdout(n) - - def read_stdout(self, n=None): - if self._r_out is not None: - data = self._r_out.read(n) - elif self._buf_stdout is None: - data = b'' - elif n is None or n == len(self._buf_stdout): - data = self._buf_stdout - self._buf_stdout = None - else: - data = self._buf_stdout[:n] - self._buf_stdout = self._buf_stdout[n:] - return data - - def read_stderr(self, n=None): - if self._combined: - return b'' - if self._r_err is not None: - data = self._r_err.read(n) - elif self._buf_stderr is None: - data = b'' - elif n is None or n == len(self._buf_stderr): - data = self._buf_stderr - self._buf_stderr = None - else: - data = self._buf_stderr[:n] - self._buf_stderr = self._buf_stderr[n:] - return data +class CapturedResults: + STDIO = dedent("""\ + import contextlib, io + with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}: + _captured_std{stream} = io.StringIO() + with contextlib.redirect_std{stream}(_captured_std{stream}): + ######################### + # begin wrapped script -class WatchedScript: - """ - Embeds a script in a new script that identifies when the script finishes. - Captures any uncaught exception, and uses a pipe to expose it. - """ + {indented} - WRAPPER = dedent(""" - import os, traceback, json - w_done = {w_done} - try: - os.write(w_done, b'\0') # started - except OSError: - # It was canceled. - pass - else: + # end wrapped script + ######################### + text = _captured_std{stream}.getvalue() + _spipe_{stream}.write(text.encode('utf-8')) + """)[:-1] + EXC = dedent("""\ + import json, traceback + with open({w_pipe}, 'wb', buffering=0) as _spipe_exc: try: ######################### # begin wrapped script - {wrapped} + {indented} # end wrapped script ######################### except Exception as exc: + # This matches what _interpreters.exec() returns. text = json.dumps(dict( type=dict( __name__=type(exc).__name__, @@ -389,41 +94,86 @@ class WatchedScript: formatted=traceback.format_exception_only(exc), errdisplay=traceback.format_exception(exc), )) - try: - os.write(w_done, text.encode('utf-8')) - except BrokenPipeError: - # It was closed already. - pass - except OSError: - if exc.errno != 9: - raise # re-raise - # It was closed already. - finally: - try: - os.close({w_done}) - except OSError: - if exc.errno != 9: - raise # re-raise - # It was closed already. - """) - - def __init__(self, script): - self._started = False - self._finished = False - self._r_done, self._w_done = os.pipe() - - wrapper = self.WRAPPER.format( - w_done=self._w_done, - wrapped=indent(script, ' '), - ) - t = threading.Thread(target=self._watch) - t.start() - - self._script = wrapper - self._thread = t - - def __del__(self): - self.close() + _spipe_exc.write(text.encode('utf-8')) + """)[:-1] + + @classmethod + def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): + script = dedent(script).strip(os.linesep) + wrapped = script + + # Handle exc. + if exc: + exc = os.pipe() + r_exc, w_exc = exc + indented = wrapped.replace('\n', '\n ') + wrapped = cls.EXC.format( + w_pipe=w_exc, + indented=indented, + ) + else: + exc = None + + # Handle stdout. + if stdout: + stdout = os.pipe() + r_out, w_out = stdout + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_out, + indented=indented, + stream='out', + ) + else: + stdout = None + + # Handle stderr. + if stderr == 'stdout': + stderr = None + elif stderr: + stderr = os.pipe() + r_err, w_err = stderr + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_err, + indented=indented, + stream='err', + ) + else: + stderr = None + + if wrapped == script: + raise NotImplementedError + results = cls(stdout, stderr, exc) + return wrapped, results + + def __init__(self, out, err, exc): + self._rf_out = None + self._rf_err = None + self._rf_exc = None + self._w_out = None + self._w_err = None + self._w_exc = None + + if out is not None: + r_out, w_out = out + self._rf_out = open(r_out, 'rb', buffering=0) + self._w_out = w_out + + if err is not None: + r_err, w_err = err + self._rf_err = open(r_err, 'rb', buffering=0) + self._w_err = w_err + + if exc is not None: + r_exc, w_exc = exc + self._rf_exc = open(r_exc, 'rb', buffering=0) + self._w_exc = w_exc + + self._buf_out = b'' + self._buf_err = b'' + self._buf_exc = b'' + self._exc = None def __enter__(self): return self @@ -431,66 +181,119 @@ def __enter__(self): def __exit__(self, *args): self.close() - @property - def script(self): - return self._script - def close(self): - for fd in [self._r_done, self._w_done]: - if fd is not None: - try: - os.close(fd) - except OSError: - if exc.errno != 9: - raise # re-raise - # It was closed already. - self._r_done = None - self._w_done = None - - @property - def finished(self): - if isinstance(self._finished, dict): - exc = types.SimpleNamespace(**self._finished) - exc.type = types.SimpleNamespace(**exc.type) - return exc - return self._finished - - def _watch(self): - r_fd = self._r_done - - assert not self._started + if self._w_out is not None: + _close_file(self._w_out) + self._w_out = None + if self._w_err is not None: + _close_file(self._w_err) + self._w_err = None + if self._w_exc is not None: + _close_file(self._w_exc) + self._w_exc = None + + self._capture() + + if self._rf_out is not None: + _close_file(self._rf_out) + self._rf_out = None + if self._rf_err is not None: + _close_file(self._rf_err) + self._rf_err = None + if self._rf_exc is not None: + _close_file(self._rf_exc) + self._rf_exc = None + + def _capture(self): + # Ideally this is called only after the script finishes + # (and thus has closed the write end of the pipe. + if self._rf_out is not None: + chunk = self._rf_out.read(100) + while chunk: + self._buf_out += chunk + chunk = self._rf_out.read(100) + if self._rf_err is not None: + chunk = self._rf_err.read(100) + while chunk: + self._buf_err += chunk + chunk = self._rf_err.read(100) + if self._rf_exc is not None: + chunk = self._rf_exc.read(100) + while chunk: + self._buf_exc += chunk + chunk = self._rf_exc.read(100) + + def stdout(self): + self._capture() + return self._buf_out.decode('utf-8') + + def stderr(self): + self._capture() + return self._buf_err.decode('utf-8') + + def exc(self): + if self._exc is not None: + return self._exc + self._capture() + if not self._buf_exc: + return None try: - ch0 = os.read(r_fd, 1) - except OSError as exc: - if exc.errno != 9: - raise # re-raise - # It was closed already. - return - if ch0 == b'': - # The write end of the pipe has closed. - return - assert ch0 == b'\0', repr(ch0) - self._started = True + data = json.loads(self._buf_exc) + except json.decoder.JSONDecodeError: + warnings.warn('incomplete exception data', RuntimeWarning) + print(self._buf_exc.decode('utf-8')) + return None + self._exc = exc = types.SimpleNamespace(**data) + exc.type = types.SimpleNamespace(**exc.type) + return self._exc - assert not self._finished - data = b'' + +def _captured_script(script, *, stdout=True, stderr=False, exc=False): + return CapturedResults.wrap_script( + script, + stdout=stdout, + stderr=stderr, + exc=exc, + ) + + +def clean_up_interpreters(): + for interp in interpreters.list_all(): + if interp.id == 0: # main + continue try: - chunk = os.read(r_fd, 100) - self._finished = True - while chunk: - data += chunk - chunk = os.read(r_fd, 100) - os.close(r_fd) - except OSError as exc: - if exc.errno != 9: - raise # re-raise - # It was closed already. + interp.close() + except RuntimeError: + pass # already destroyed + + +def _run_output(interp, request, init=None): + script, (stdout, _, _) = _captured_script(request) + with stdout: + if init: + interp.prepare_main(init) + interp.exec(script) + return stdout.read() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interp.exec(dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) - if data: - self._finished = json.loads(data) + t = threading.Thread(target=run) + t.start() - # It may be useful to implement the concurrent.futures.Future API - # on this class. + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() class TestBase(unittest.TestCase): @@ -603,14 +406,38 @@ def assert_ns_equal(self, ns1, ns2, msg=None): standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) + def _run_string(self, interp, script): + wrapped, results = _captured_script(script, exc=False) + #_dump_script(wrapped) + with results: + if isinstance(interp, interpreters.Interpreter): + interp.exec(script) + else: + err = _interpreters.run_string(interp, wrapped) + if err is not None: + return None, err + return results.stdout(), None + + def run_and_capture(self, interp, script): + text, err = self._run_string(interp, script) + if err is not None: + print() + if not err.errdisplay.startswith('Traceback '): + print('Traceback (most recent call last):') + print(err.errdisplay, file=sys.stderr) + raise Exception(f'subinterpreter failed: {err.formatted}') + else: + return text + @requires__testinternalcapi def run_external(self, script, config='legacy'): - with CapturingScript(script, combined=False) as captured: - with WatchedScript(captured.script) as watched: - rc = run_in_interpreter(watched.script, config) - assert rc == 0, rc - text = watched.read(100).decode('utf-8') - err = watched.finished - if err is True: - err = None + if isinstance(config, str): + config = _interpreters.new_config(config) + wrapped, results = _captured_script(script, exc=True) + #_dump_script(wrapped) + with results: + rc = run_in_interpreter(wrapped, config) + assert rc == 0, rc + text = results.stdout() + err = results.exc() return err, text From 0c24e5ccd0ec1c93dde9891cf33cab7e01578a77 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 28 Mar 2024 16:50:57 -0600 Subject: [PATCH 10/28] Finish the tests. --- Lib/test/test_interpreters/test_api.py | 105 +++++++++++++++++++++---- Lib/test/test_interpreters/utils.py | 6 +- 2 files changed, 93 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 445a81211d6a4e..25b80908b20eab 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -12,7 +12,10 @@ _interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError -from .utils import _captured_script, _run_output, _running, TestBase +from .utils import ( + _captured_script, _run_output, _running, TestBase, + requires__testinternalcapi, _testinternalcapi, +) class ModuleTests(TestBase): @@ -158,19 +161,21 @@ def test_idempotent(self): id2 = id(interp) self.assertNotEqual(id1, id2) + @requires__testinternalcapi def test_unmanaged(self): - with self.unmanaged_interpreter() as unmanaged: - print(unmanaged) - err = unmanaged.exec(dedent(f""" - import {interpreters.__name__} as interpreters - err = None - try: - interpreters.get_current() - except ValueError as exc: - err = exc - assert exc is not None - """)) - self.assertEqual(err, '') + last = 0 + for id, *_ in _interpreters.list_all(): + last = max(last, id) + expected = _testinternalcapi.next_interpreter_id() + err, text = self.run_external(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print((interp.id, interp.owned)) + """) + assert err is None, err + interpid, owned = eval(text) + self.assertEqual(interpid, expected) + self.assertFalse(owned) class ListAllTests(TestBase): @@ -215,7 +220,32 @@ def test_idempotent(self): self.assertIs(interp1, interp2) def test_unmanaged(self): - ... + mainid, _ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected = [ + (mainid, False), + (interpid1, True), + (interpid2, True), + (interpid3, True), + (interpid4, False), + (interpid5, True), + ] + expected2 = expected[:-2] + err, text = self.run_external(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.create() + print( + [(i.id, i.owned) for i in interpreters.list_all()]) + """) + assert err is None, err + res = eval(text) + res2 = [(i.id, i.owned) for i in interpreters.list_all()] + self.assertEqual(res, expected) + self.assertEqual(res2, expected2) class InterpreterObjectTests(TestBase): @@ -1138,7 +1168,52 @@ def parse_stdout(text): self.assertFalse(owned) def test_list_all(self): - ... + mainid, _ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + expected = [ + (mainid, False), + (interpid1, True), + (interpid2, True), + (interpid3, True), + ] + + with self.subTest('main'): + res = _interpreters.list_all() + self.assertEqual(res, expected) + + with self.subTest('owned'): + text = self.run_and_capture(interpid2, f""" + import {_interpreters.__name__} as _interpreters + print( + _interpreters.list_all()) + """) + + res = eval(text) + self.assertEqual(res, expected) + + with self.subTest('external'): + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected2 = expected + [ + (interpid4, False), + (interpid5, True), + ] + expected3 = expected + [ + (interpid5, True), + ] + err, text = self.run_external(f""" + import {_interpreters.__name__} as _interpreters + _interpreters.create() + print( + _interpreters.list_all()) + """) + assert err is None, err + res2 = eval(text) + res3 = _interpreters.list_all() + self.assertEqual(res2, expected2) + self.assertEqual(res3, expected3) def test_create(self): isolated = _interpreters.new_config('isolated') diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 7eb65baf681535..9078e791812019 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -268,12 +268,12 @@ def clean_up_interpreters(): def _run_output(interp, request, init=None): - script, (stdout, _, _) = _captured_script(request) - with stdout: + script, results = _captured_script(request) + with results: if init: interp.prepare_main(init) interp.exec(script) - return stdout.read() + return results.stdout() @contextlib.contextmanager From 611fa31a426a78070602b4f462b1d1076e0805f1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 1 Apr 2024 13:42:45 -0600 Subject: [PATCH 11/28] Add missing tests. --- Lib/test/test_interpreters/test_api.py | 88 +++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 25b80908b20eab..9f7f68bca06fb6 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1220,17 +1220,17 @@ def test_create(self): legacy = _interpreters.new_config('legacy') default = isolated - with self.subTest('no arg'): + with self.subTest('no args'): interpid = _interpreters.create() config = _interpreters.get_config(interpid) self.assert_ns_equal(config, default) - with self.subTest('arg: None'): + with self.subTest('config: None'): interpid = _interpreters.create(None) config = _interpreters.get_config(interpid) self.assert_ns_equal(config, default) - with self.subTest('arg: \'empty\''): + with self.subTest('config: \'empty\''): with self.assertRaises(interpreters.InterpreterError): # The "empty" config isn't viable on its own. _interpreters.create('empty') @@ -1266,6 +1266,17 @@ def test_create(self): with self.assertRaises(ValueError): _interpreters.create(orig) + def test_destroy(self): + interpid = _interpreters.create() + before = [id for id, _ in _interpreters.list_all()] + _interpreters.destroy(interpid) + after = [id for id, _ in _interpreters.list_all()] + + self.assertIn(interpid, before) + self.assertNotIn(interpid, after) + with self.assertRaises(interpreters.InterpreterNotFoundError): + _interpreters.is_running(interpid) + def test_get_config(self): with self.subTest('main'): expected = _interpreters.new_config('legacy') @@ -1286,6 +1297,77 @@ def test_get_config(self): config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) + def test_exec(self): + with self.subTest('run script'): + interpid = _interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + exc = _interpreters.exec(interpid, script) + out = results.stdout() + self.assertIs(exc, None) + self.assertEqual(out, 'it worked!') + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + script, results = _captured_script(""" + raise Exception('uh-oh!') + print("it worked!", end="") + """) + with results: + exc = _interpreters.exec(interpid, script) + out = results.stdout() + self.assertEqual(out, '') + self.assertEqual(exc, types.SimpleNamespace( + type=types.SimpleNamespace( + __name__='Exception', + __qualname__='Exception', + __module__='builtins', + ), + msg='uh-oh!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + )) + + def test_call(self): + with self.subTest('no args'): + interpid = _interpreters.create() + exc = _interpreters.call(interpid, call_func_return_shareable) + self.assertIs(exc, None) + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + exc = _interpreters.call(interpid, call_func_failure) + self.assertEqual(exc, types.SimpleNamespace( + type=types.SimpleNamespace( + __name__='Exception', + __qualname__='Exception', + __module__='builtins', + ), + msg='spam!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + )) + + def test_set___main___attrs(self): + interpid = _interpreters.create() + before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + before2 = _interpreters.exec(interpid, 'assert ham == 42') + self.assertEqual(before1.type.__name__, 'NameError') + self.assertEqual(before2.type.__name__, 'NameError') + + _interpreters.set___main___attrs(interpid, dict( + spam='eggs', + ham=42, + )) + after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + after2 = _interpreters.exec(interpid, 'assert ham == 42') + after3 = _interpreters.exec(interpid, 'assert spam == 42') + self.assertIs(after1, None) + self.assertIs(after2, None) + self.assertEqual(after3.type.__name__, 'AssertionError') + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. From bdc09f9ec984641aabcdc55d35c85c3baf7c4e35 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 10:43:26 -0600 Subject: [PATCH 12/28] Add more capture/exec helpers. --- Lib/test/test_interpreters/utils.py | 203 ++++++++++++++++++++++++++-- 1 file changed, 189 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 9078e791812019..bf79a5b6508d24 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -1,3 +1,4 @@ +from collections import namedtuple import contextlib import json import io @@ -36,6 +37,7 @@ def requires__testinternalcapi(func): def _dump_script(text): lines = text.splitlines() + print() print('-' * 20) for i, line in enumerate(lines, 1): print(f' {i:>{len(str(len(lines)))}} {line}') @@ -54,7 +56,7 @@ def _close_file(file): # It was closed already. -class CapturedResults: +class CapturingResults: STDIO = dedent("""\ import contextlib, io @@ -175,13 +177,23 @@ def __init__(self, out, err, exc): self._buf_exc = b'' self._exc = None + self._closed = False + def __enter__(self): return self def __exit__(self, *args): self.close() + @property + def closed(self): + return self._closed + def close(self): + if self._closed: + return + self._closed = True + if self._w_out is not None: _close_file(self._w_out) self._w_out = None @@ -223,18 +235,15 @@ def _capture(self): self._buf_exc += chunk chunk = self._rf_exc.read(100) - def stdout(self): - self._capture() + def _unpack_stdout(self): return self._buf_out.decode('utf-8') - def stderr(self): - self._capture() + def _unpack_stderr(self): return self._buf_err.decode('utf-8') - def exc(self): + def _unpack_exc(self): if self._exc is not None: return self._exc - self._capture() if not self._buf_exc: return None try: @@ -247,9 +256,66 @@ def exc(self): exc.type = types.SimpleNamespace(**exc.type) return self._exc + def stdout(self): + if self.closed: + return self.final().stdout + self._capture() + return self._unpack_stdout() + + def stderr(self): + if self.closed: + return self.final().stderr + self._capture() + return self._unpack_stderr() + + def exc(self): + if self.closed: + return self.final().exc + self._capture() + return self._unpack_exc() + + def final(self, *, force=False): + try: + return self._final + except AttributeError: + if not self._closed: + if not force: + raise Exception('no final results available yet') + else: + return CapturedResults.Proxy(self) + self._final = CapturedResults( + self._unpack_stdout(), + self._unpack_stderr(), + self._unpack_exc(), + ) + return self._final + + +class CapturedResults(namedtuple('CapturedResults', 'stdout stderr exc')): + + class Proxy: + def __init__(self, capturing): + self._capturing = capturing + def _finish(self): + if self._capturing is None: + return + self._final = self._capturing.final() + self._capturing = None + def __iter__(self): + self._finish() + yield from self._final + def __len__(self): + self._finish() + return len(self._final) + def __getattr__(self, name): + self._finish() + if name.startswith('_'): + raise AttributeError(name) + return getattr(self._final, name) + def _captured_script(script, *, stdout=True, stderr=False, exc=False): - return CapturedResults.wrap_script( + return CapturingResults.wrap_script( script, stdout=stdout, stderr=stderr, @@ -430,14 +496,123 @@ def run_and_capture(self, interp, script): return text @requires__testinternalcapi - def run_external(self, script, config='legacy'): + @contextlib.contextmanager + def unmanaged_interpreter(self, config='legacy'): if isinstance(config, str): config = _interpreters.new_config(config) - wrapped, results = _captured_script(script, exc=True) + interpid = _testinternalcapi.create_interpreter() + try: + yield interpid + finally: + try: + _testinternalcapi.destroy_interpreter(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + @contextlib.contextmanager + def capturing(self, script): + wrapped, capturing = _captured_script(script, stdout=True, exc=True) #_dump_script(wrapped) - with results: + with capturing: + yield wrapped, capturing.final(force=True) + + @requires__testinternalcapi + def run_external(self, interpid, script, *, main=False): + with self.capturing(script) as (wrapped, results): + rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main) + assert rc == 0, rc + return results.exc, results.stdout + + @contextlib.contextmanager + def _running(self, run_interp, exec_interp): + token = b'\0' + r_in, w_in = self.pipe() + r_out, w_out = self.pipe() + + def close(): + _close_file(r_in) + _close_file(w_in) + _close_file(r_out) + _close_file(w_out) + + # Start running (and wait). + script = dedent(f""" + import os + # handshake + token = os.read({r_in}, 1) + os.write({w_out}, token) + # Wait for the "done" message. + os.read({r_in}, 1) + """) + failed = None + def run(): + nonlocal failed + try: + run_interp(script) + except Exception as exc: + failed = exc + close() + t = threading.Thread(target=run) + t.start() + + # handshake + try: + os.write(w_in, token) + token2 = os.read(r_out, 1) + assert token2 == token, (token2, token) + except OSError: + t.join() + if failed is not None: + raise failed + + # CM __exit__() + try: + try: + yield + finally: + # Send "done". + os.write(w_in, b'\0') + finally: + close() + t.join() + if failed is not None: + raise failed + + @contextlib.contextmanager + def running(self, interp): + if isinstance(interp, int): + interpid = interp + def exec_interp(script): + exc = _interpreters.exec(interpid, script) + assert exc is None, exc + run_interp = exec_interp + else: + def run_interp(script): + text = self.run_and_capture(interp, script) + assert text == '', repr(text) + def exec_interp(script): + interp.exec(script) + with self._running(run_interp, exec_interp): + yield + + @requires__testinternalcapi + @contextlib.contextmanager + def running_external(self, interpid, *, main=False): + def run_interp(script): + err, text = self.run_external(interpid, script, main=main) + assert err is None, err + assert text == '', repr(text) + def exec_interp(script): + rc = _testinternalcapi.exec_interpreter(interpid, script) + assert rc == 0, rc + with self._running(run_interp, exec_interp): + yield + + @requires__testinternalcapi + def run_temp_external(self, script, config='legacy'): + if isinstance(config, str): + config = _interpreters.new_config(config) + with self.capturing(script) as (wrapped, results): rc = run_in_interpreter(wrapped, config) assert rc == 0, rc - text = results.stdout() - err = results.exc() - return err, text + return results.exc, results.stdout From be85ff510af8d1e6dfecb70218676dc6913ee8b8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 15:41:11 -0600 Subject: [PATCH 13/28] Fill out the tests. --- Lib/test/test_interpreters/test_api.py | 308 +++++++++++++++++++++---- 1 file changed, 267 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 9f7f68bca06fb6..f6a20d08fe3b88 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -11,7 +11,9 @@ # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters -from test.support.interpreters import InterpreterNotFoundError +from test.support.interpreters import ( + InterpreterError, InterpreterNotFoundError, ExecutionFailed, +) from .utils import ( _captured_script, _run_output, _running, TestBase, requires__testinternalcapi, _testinternalcapi, @@ -162,12 +164,12 @@ def test_idempotent(self): self.assertNotEqual(id1, id2) @requires__testinternalcapi - def test_unmanaged(self): + def test_not_owned(self): last = 0 for id, *_ in _interpreters.list_all(): last = max(last, id) expected = _testinternalcapi.next_interpreter_id() - err, text = self.run_external(f""" + err, text = self.run_temp_external(f""" import {interpreters.__name__} as interpreters interp = interpreters.get_current() print((interp.id, interp.owned)) @@ -219,7 +221,7 @@ def test_idempotent(self): for interp1, interp2 in zip(actual, expected): self.assertIs(interp1, interp2) - def test_unmanaged(self): + def test_not_owned(self): mainid, _ = _interpreters.get_main() interpid1 = _interpreters.create() interpid2 = _interpreters.create() @@ -235,7 +237,7 @@ def test_unmanaged(self): (interpid5, True), ] expected2 = expected[:-2] - err, text = self.run_external(f""" + err, text = self.run_temp_external(f""" import {interpreters.__name__} as interpreters interp = interpreters.create() print( @@ -299,6 +301,33 @@ def test_id_readonly(self): with self.assertRaises(AttributeError): interp.id = 1_000_000 + def test_owned(self): + main = interpreters.get_main() + interp = interpreters.create() + + with self.subTest('main'): + self.assertFalse(main.owned) + + with self.subTest('owned'): + self.assertTrue(interp.owned) + + with self.subTest('not owned'): + err, text = self.run_temp_external(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(interp.owned) + """) + self.assertIsNone(err) + owned = eval(text) + self.assertTrue(owned) + + with self.subTest('readonly'): + for value in (True, False): + with self.assertRaises(AttributeError): + interp.id = value + with self.assertRaises(AttributeError): + main.id = value + def test_hashable(self): interp = interpreters.create() expected = hash(interp.id) @@ -324,6 +353,7 @@ def test_main(self): main = interpreters.get_main() self.assertTrue(main.is_running()) + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_subinterpreter(self): interp = interpreters.create() @@ -385,6 +415,59 @@ def task(): interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), FINISHED) + def test_not_owned(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(interp.is_running()) + """) + def resolve_results(err, text): + assert err is None, err + self.assertNotEqual(text, "") + try: + return eval(text) + except Exception: + raise Exception(repr(text)) + + with self.subTest('running __main__ (from self)'): + with self.unmanaged_interpreter() as interpid: + err, text = self.run_external(interpid, script, main=True) + running = resolve_results(err, text) + self.assertTrue(running) + + with self.subTest('running, but not __main__ (from self)'): + err, text = self.run_temp_external(script) + running = resolve_results(err, text) + self.assertFalse(running) + + with self.subTest('running __main__ (from other)'): + with self.unmanaged_interpreter() as interpid: + interp = interpreters.Interpreter(interpid, _owned=False) + before = interp.is_running() + with self.running_external(interpid, main=True): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertTrue(during) + self.assertFalse(after) + + with self.subTest('running, but not __main__ (from other)'): + with self.unmanaged_interpreter() as interpid: + interp = interpreters.Interpreter(interpid, _owned=False) + before = interp.is_running() + with self.running_external(interpid, main=False): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertFalse(during) + self.assertFalse(after) + + with self.subTest('not running (from other)'): + with self.unmanaged_interpreter() as interpid: + interp = interpreters.Interpreter(interpid, _owned=False) + running = interp.is_running() + self.assertFalse(running) + class TestInterpreterClose(TestBase): @@ -412,11 +495,11 @@ def test_all(self): def test_main(self): main, = interpreters.list_all() - with self.assertRaises(interpreters.InterpreterError): + with self.assertRaises(InterpreterError): main.close() def f(): - with self.assertRaises(interpreters.InterpreterError): + with self.assertRaises(InterpreterError): main.close() t = threading.Thread(target=f) @@ -467,12 +550,13 @@ def f(): t.start() t.join() + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_still_running(self): main, = interpreters.list_all() interp = interpreters.create() with _running(interp): - with self.assertRaises(interpreters.InterpreterError): + with self.assertRaises(InterpreterError): interp.close() self.assertTrue(interp.is_running()) @@ -507,6 +591,57 @@ def task(): self.assertEqual(os.read(r_interp, 1), FINISHED) + def test_not_owned(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + interp.close() + """) + def check_results(err, text): + self.assertIsNot(err, None) + self.assertEqual(err.type.__name__, 'InterpreterError') + self.assertIn('current', err.msg) + self.assertEqual(text, '') + + with self.subTest('running __main__ (from self)'): + with self.unmanaged_interpreter() as interpid: + err, text = self.run_external(interpid, script, main=True) + check_results(err, text) + + with self.subTest('running, but not __main__ (from self)'): + err, text = self.run_temp_external(script) + check_results(err, text) + + with self.subTest('running __main__ (from other)'): + with self.unmanaged_interpreter() as interpid: + interp = interpreters.Interpreter(interpid, _owned=False) + with self.running_external(interpid, main=True): + with self.assertRaisesRegex(InterpreterError, 'running'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue(interp.is_running()) + + # The rest must be skipped until we deal with running threads when + # interp.close() is called. + return + + with self.subTest('running, but not __main__ (from other)'): + with self.unmanaged_interpreter() as interpid: + interp = interpreters.Interpreter(interpid, _owned=False) + with self.running_external(interpid, main=False): + with self.assertRaisesRegex(InterpreterError, 'not managed'): + interp.close() + # Make sure it wssn't closed. + self.assertFalse(interp.is_running()) + + with self.subTest('not running (from other)'): + with self.unmanaged_interpreter() as interpid: + interp = interpreters.Interpreter(interpid, _owned=False) + with self.assertRaisesRegex(InterpreterError, 'not managed'): + interp.close() + # Make sure it wssn't closed. + self.assertFalse(interp.is_running()) + class TestInterpreterPrepareMain(TestBase): @@ -559,11 +694,28 @@ def test_not_shareable(self): interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) # Make sure neither was actually bound. - with self.assertRaises(interpreters.ExecutionFailed): + with self.assertRaises(ExecutionFailed): interp.exec('print(foo)') - with self.assertRaises(interpreters.ExecutionFailed): + with self.assertRaises(ExecutionFailed): interp.exec('print(spam)') + def test_running(self): + interp = interpreters.create() + interp.prepare_main({'spam': True}) + with self.running(interp): + with self.assertRaisesRegex(InterpreterError, 'running'): + interp.prepare_main({'spam': False}) + interp.exec('assert spam is True') + + @requires__testinternalcapi + def test_not_owned(self): + with self.unmanaged_interpreter() as interpid: + interp = interpreters.Interpreter(interpid, _owned=False) + interp.prepare_main({'spam': True}) + rc = _testinternalcapi.exec_interpreter(interpid, + 'assert spam is True') + assert rc == 0, rc + class TestInterpreterExec(TestBase): @@ -578,7 +730,7 @@ def test_success(self): def test_failure(self): interp = interpreters.create() - with self.assertRaises(interpreters.ExecutionFailed): + with self.assertRaises(ExecutionFailed): interp.exec('raise Exception') def test_display_preserved_exception(self): @@ -666,6 +818,7 @@ def test_fork(self): content = file.read() self.assertEqual(content, expected) + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_already_running(self): interp = interpreters.create() @@ -714,6 +867,13 @@ def task(): self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), FINISHED) + @requires__testinternalcapi + def test_not_owned(self): + with self.unmanaged_interpreter() as interpid: + interp = interpreters.Interpreter(interpid, _owned=False) + with self.assertRaisesRegex(ExecutionFailed, 'it worked'): + interp.exec('raise Exception("it worked!")') + # test_xxsubinterpreters covers the remaining # Interpreter.exec() behavior. @@ -878,7 +1038,7 @@ def test_call(self): raise Exception((args, kwargs)) interp.call(callable) - with self.assertRaises(interpreters.ExecutionFailed): + with self.assertRaises(ExecutionFailed): interp.call(call_func_failure) def test_call_in_thread(self): @@ -1156,12 +1316,12 @@ def parse_stdout(text): self.assertEqual(interpid, orig) self.assertTrue(owned) - with self.subTest('external'): + with self.subTest('not owned'): last = 0 for id, *_ in _interpreters.list_all(): last = max(last, id) expected = last + 1 - err, text = self.run_external(script) + err, text = self.run_temp_external(script) assert err is None, err interpid, owned = parse_stdout(text) self.assertEqual(interpid, expected) @@ -1193,7 +1353,7 @@ def test_list_all(self): res = eval(text) self.assertEqual(res, expected) - with self.subTest('external'): + with self.subTest('not owned'): interpid4 = interpid3 + 1 interpid5 = interpid4 + 1 expected2 = expected + [ @@ -1203,7 +1363,7 @@ def test_list_all(self): expected3 = expected + [ (interpid5, True), ] - err, text = self.run_external(f""" + err, text = self.run_temp_external(f""" import {_interpreters.__name__} as _interpreters _interpreters.create() print( @@ -1231,7 +1391,7 @@ def test_create(self): self.assert_ns_equal(config, default) with self.subTest('config: \'empty\''): - with self.assertRaises(interpreters.InterpreterError): + with self.assertRaises(InterpreterError): # The "empty" config isn't viable on its own. _interpreters.create('empty') @@ -1267,15 +1427,28 @@ def test_create(self): _interpreters.create(orig) def test_destroy(self): - interpid = _interpreters.create() - before = [id for id, _ in _interpreters.list_all()] - _interpreters.destroy(interpid) - after = [id for id, _ in _interpreters.list_all()] + with self.subTest('owned'): + interpid = _interpreters.create() + before = [id for id, _ in _interpreters.list_all()] + _interpreters.destroy(interpid) + after = [id for id, _ in _interpreters.list_all()] - self.assertIn(interpid, before) - self.assertNotIn(interpid, after) - with self.assertRaises(interpreters.InterpreterNotFoundError): - _interpreters.is_running(interpid) + self.assertIn(interpid, before) + self.assertNotIn(interpid, after) + with self.assertRaises(InterpreterNotFoundError): + _interpreters.is_owned(interpid) + + with self.subTest('main'): + interpid, _ = _interpreters.get_main() + with self.assertRaises(InterpreterError): + # It is the current interpreter. + _interpreters.destroy(interpid) + + with self.subTest('not owned'): + interpid = _testinternalcapi.create_interpreter() + _interpreters.destroy(interpid) + with self.assertRaises(InterpreterNotFoundError): + _interpreters.is_running(interpid) def test_get_config(self): with self.subTest('main'): @@ -1297,6 +1470,46 @@ def test_get_config(self): config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) + with self.subTest('not owned'): + orig = _interpreters.new_config('isolated') + with self.unmanaged_interpreter(orig) as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + def test_is_running(self): + with self.subTest('main'): + interpid, _ = _interpreters.get_main() + running = _interpreters.is_running(interpid) + self.assertTrue(running) + + with self.subTest('owned (running)'): + interpid = _interpreters.create() + with self.running(interpid): + running = _interpreters.is_running(interpid) + self.assertTrue(running) + + with self.subTest('owned (not running)'): + interpid = _interpreters.create() + running = _interpreters.is_running(interpid) + self.assertFalse(running) + + with self.subTest('not owned (running __main__)'): + with self.unmanaged_interpreter() as interpid: + with self.running_external(interpid, main=True): + running = _interpreters.is_running(interpid) + self.assertTrue(running) + + with self.subTest('not owned (running, but not __main__)'): + with self.unmanaged_interpreter() as interpid: + with self.running_external(interpid, main=False): + running = _interpreters.is_running(interpid) + self.assertFalse(running) + + with self.subTest('not owned (not running)'): + with self.unmanaged_interpreter() as interpid: + running = _interpreters.is_running(interpid) + self.assertFalse(running) + def test_exec(self): with self.subTest('run script'): interpid = _interpreters.create() @@ -1329,6 +1542,12 @@ def test_exec(self): errdisplay=exc.errdisplay, )) + with self.subTest('not owned'): + with self.unmanaged_interpreter() as interpid: + exc = _interpreters.exec(interpid, 'raise Exception("it worked!")') + self.assertIsNot(exc, None) + self.assertEqual(exc.msg, 'it worked!') + def test_call(self): with self.subTest('no args'): interpid = _interpreters.create() @@ -1351,22 +1570,29 @@ def test_call(self): )) def test_set___main___attrs(self): - interpid = _interpreters.create() - before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') - before2 = _interpreters.exec(interpid, 'assert ham == 42') - self.assertEqual(before1.type.__name__, 'NameError') - self.assertEqual(before2.type.__name__, 'NameError') - - _interpreters.set___main___attrs(interpid, dict( - spam='eggs', - ham=42, - )) - after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') - after2 = _interpreters.exec(interpid, 'assert ham == 42') - after3 = _interpreters.exec(interpid, 'assert spam == 42') - self.assertIs(after1, None) - self.assertIs(after2, None) - self.assertEqual(after3.type.__name__, 'AssertionError') + with self.subTest('owned'): + interpid = _interpreters.create() + before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + before2 = _interpreters.exec(interpid, 'assert ham == 42') + self.assertEqual(before1.type.__name__, 'NameError') + self.assertEqual(before2.type.__name__, 'NameError') + + _interpreters.set___main___attrs(interpid, dict( + spam='eggs', + ham=42, + )) + after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + after2 = _interpreters.exec(interpid, 'assert ham == 42') + after3 = _interpreters.exec(interpid, 'assert spam == 42') + self.assertIs(after1, None) + self.assertIs(after2, None) + self.assertEqual(after3.type.__name__, 'AssertionError') + + with self.subTest('not owned'): + with self.unmanaged_interpreter() as interpid: + _interpreters.set___main___attrs(interpid, {'spam': True}) + exc = _interpreters.exec(interpid, 'assert spam is True') + self.assertIsNone(exc) if __name__ == '__main__': From 51f18f875ddd3273e261c0d7a7d037e1d2052f3c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 17:21:28 -0600 Subject: [PATCH 14/28] Adjust the tests. --- Lib/test/support/interpreters/__init__.py | 7 +- Lib/test/test_interpreters/test_api.py | 212 +++++++++------------- Lib/test/test_interpreters/utils.py | 5 + Modules/_xxsubinterpretersmodule.c | 36 ++-- 4 files changed, 122 insertions(+), 138 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 8be4ee736aa93b..60323c9874f9a0 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -79,18 +79,19 @@ def create(): def list_all(): """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] + return [Interpreter(id) + for id, in _interpreters.list_all()] def get_current(): """Return the currently running interpreter.""" - id = _interpreters.get_current() + id, = _interpreters.get_current() return Interpreter(id) def get_main(): """Return the main interpreter.""" - id = _interpreters.get_main() + id, = _interpreters.get_main() return Interpreter(id) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index f6a20d08fe3b88..484e16ebb73133 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -164,7 +164,7 @@ def test_idempotent(self): self.assertNotEqual(id1, id2) @requires__testinternalcapi - def test_not_owned(self): + def test_created_with_capi(self): last = 0 for id, *_ in _interpreters.list_all(): last = max(last, id) @@ -172,12 +172,11 @@ def test_not_owned(self): err, text = self.run_temp_external(f""" import {interpreters.__name__} as interpreters interp = interpreters.get_current() - print((interp.id, interp.owned)) + print(interp.id) """) assert err is None, err - interpid, owned = eval(text) + interpid = eval(text) self.assertEqual(interpid, expected) - self.assertFalse(owned) class ListAllTests(TestBase): @@ -221,31 +220,31 @@ def test_idempotent(self): for interp1, interp2 in zip(actual, expected): self.assertIs(interp1, interp2) - def test_not_owned(self): - mainid, _ = _interpreters.get_main() + def test_created_with_capi(self): + mainid, *_ = _interpreters.get_main() interpid1 = _interpreters.create() interpid2 = _interpreters.create() interpid3 = _interpreters.create() interpid4 = interpid3 + 1 interpid5 = interpid4 + 1 expected = [ - (mainid, False), - (interpid1, True), - (interpid2, True), - (interpid3, True), - (interpid4, False), - (interpid5, True), + (mainid,), + (interpid1,), + (interpid2,), + (interpid3,), + (interpid4,), + (interpid5,), ] expected2 = expected[:-2] err, text = self.run_temp_external(f""" import {interpreters.__name__} as interpreters interp = interpreters.create() print( - [(i.id, i.owned) for i in interpreters.list_all()]) + [(i.id,) for i in interpreters.list_all()]) """) assert err is None, err res = eval(text) - res2 = [(i.id, i.owned) for i in interpreters.list_all()] + res2 = [(i.id,) for i in interpreters.list_all()] self.assertEqual(res, expected) self.assertEqual(res2, expected2) @@ -301,33 +300,6 @@ def test_id_readonly(self): with self.assertRaises(AttributeError): interp.id = 1_000_000 - def test_owned(self): - main = interpreters.get_main() - interp = interpreters.create() - - with self.subTest('main'): - self.assertFalse(main.owned) - - with self.subTest('owned'): - self.assertTrue(interp.owned) - - with self.subTest('not owned'): - err, text = self.run_temp_external(f""" - import {interpreters.__name__} as interpreters - interp = interpreters.get_current() - print(interp.owned) - """) - self.assertIsNone(err) - owned = eval(text) - self.assertTrue(owned) - - with self.subTest('readonly'): - for value in (True, False): - with self.assertRaises(AttributeError): - interp.id = value - with self.assertRaises(AttributeError): - main.id = value - def test_hashable(self): interp = interpreters.create() expected = hash(interp.id) @@ -415,7 +387,7 @@ def task(): interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), FINISHED) - def test_not_owned(self): + def test_created_with_capi(self): script = dedent(f""" import {interpreters.__name__} as interpreters interp = interpreters.get_current() @@ -432,8 +404,8 @@ def resolve_results(err, text): with self.subTest('running __main__ (from self)'): with self.unmanaged_interpreter() as interpid: err, text = self.run_external(interpid, script, main=True) - running = resolve_results(err, text) - self.assertTrue(running) + running = resolve_results(err, text) + self.assertTrue(running) with self.subTest('running, but not __main__ (from self)'): err, text = self.run_temp_external(script) @@ -441,8 +413,7 @@ def resolve_results(err, text): self.assertFalse(running) with self.subTest('running __main__ (from other)'): - with self.unmanaged_interpreter() as interpid: - interp = interpreters.Interpreter(interpid, _owned=False) + with self.unmanaged_interpreter_obj() as (interp, interpid): before = interp.is_running() with self.running_external(interpid, main=True): during = interp.is_running() @@ -452,8 +423,7 @@ def resolve_results(err, text): self.assertFalse(after) with self.subTest('running, but not __main__ (from other)'): - with self.unmanaged_interpreter() as interpid: - interp = interpreters.Interpreter(interpid, _owned=False) + with self.unmanaged_interpreter_obj() as (interp, interpid): before = interp.is_running() with self.running_external(interpid, main=False): during = interp.is_running() @@ -463,10 +433,9 @@ def resolve_results(err, text): self.assertFalse(after) with self.subTest('not running (from other)'): - with self.unmanaged_interpreter() as interpid: - interp = interpreters.Interpreter(interpid, _owned=False) + with self.unmanaged_interpreter_obj() as (interp, _): running = interp.is_running() - self.assertFalse(running) + self.assertFalse(running) class TestInterpreterClose(TestBase): @@ -591,7 +560,7 @@ def task(): self.assertEqual(os.read(r_interp, 1), FINISHED) - def test_not_owned(self): + def test_created_with_capi(self): script = dedent(f""" import {interpreters.__name__} as interpreters interp = interpreters.get_current() @@ -613,21 +582,20 @@ def check_results(err, text): check_results(err, text) with self.subTest('running __main__ (from other)'): - with self.unmanaged_interpreter() as interpid: - interp = interpreters.Interpreter(interpid, _owned=False) + with self.unmanaged_interpreter_obj() as (interp, interpid): with self.running_external(interpid, main=True): with self.assertRaisesRegex(InterpreterError, 'running'): interp.close() # Make sure it wssn't closed. - self.assertTrue(interp.is_running()) + self.assertTrue( + interp.is_running()) # The rest must be skipped until we deal with running threads when # interp.close() is called. return with self.subTest('running, but not __main__ (from other)'): - with self.unmanaged_interpreter() as interpid: - interp = interpreters.Interpreter(interpid, _owned=False) + with self.unmanaged_interpreter_obj() as (interp, interpid): with self.running_external(interpid, main=False): with self.assertRaisesRegex(InterpreterError, 'not managed'): interp.close() @@ -635,8 +603,7 @@ def check_results(err, text): self.assertFalse(interp.is_running()) with self.subTest('not running (from other)'): - with self.unmanaged_interpreter() as interpid: - interp = interpreters.Interpreter(interpid, _owned=False) + with self.unmanaged_interpreter_obj() as (interp, _): with self.assertRaisesRegex(InterpreterError, 'not managed'): interp.close() # Make sure it wssn't closed. @@ -708,9 +675,9 @@ def test_running(self): interp.exec('assert spam is True') @requires__testinternalcapi - def test_not_owned(self): + def test_created_with_capi(self): with self.unmanaged_interpreter() as interpid: - interp = interpreters.Interpreter(interpid, _owned=False) + interp = interpreters.Interpreter(interpid) interp.prepare_main({'spam': True}) rc = _testinternalcapi.exec_interpreter(interpid, 'assert spam is True') @@ -867,10 +834,8 @@ def task(): self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), FINISHED) - @requires__testinternalcapi - def test_not_owned(self): - with self.unmanaged_interpreter() as interpid: - interp = interpreters.Interpreter(interpid, _owned=False) + def test_created_with_capi(self): + with self.unmanaged_interpreter_obj() as (interp, _): with self.assertRaisesRegex(ExecutionFailed, 'it worked'): interp.exec('raise Exception("it worked!")') @@ -1150,6 +1115,14 @@ class LowLevelTests(TestBase): # encountered by the high-level module, thus they # mostly shouldn't matter as much. + def interp_exists(self, interpid): + try: + _interpreters.is_running(interpid) + except InterpreterNotFoundError: + return False + else: + return True + def test_new_config(self): # This test overlaps with # test.test_capi.test_misc.InterpreterConfigTests. @@ -1272,78 +1245,61 @@ def test_new_config(self): with self.assertRaises(ValueError): _interpreters.new_config(gil=value) - def test_get_config(self): - # This test overlaps with - # test.test_capi.test_misc.InterpreterConfigTests. - - with self.subTest('main'): - expected = _interpreters.new_config('legacy') - expected.gil = 'own' - interpid, _ = _interpreters.get_main() - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - def test_get_main(self): - interpid, owned = _interpreters.get_main() + interpid, = _interpreters.get_main() self.assertEqual(interpid, 0) - self.assertFalse(owned) def test_get_current(self): with self.subTest('main'): - main, _ = _interpreters.get_main() - interpid, owned = _interpreters.get_current() + main, *_ = _interpreters.get_main() + interpid, = _interpreters.get_current() self.assertEqual(interpid, main) - self.assertFalse(owned) script = f""" import {_interpreters.__name__} as _interpreters - interpid, owned = _interpreters.get_current() + interpid, = _interpreters.get_current() print(interpid) - print(owned) """ def parse_stdout(text): parts = text.split() - assert len(parts) == 2, parts - interpid, owned = parts + assert len(parts) == 1, parts + interpid, = parts interpid = int(interpid) - owned = eval(owned) - return interpid, owned + return interpid, - with self.subTest('owned'): + with self.subTest('from _interpreters'): orig = _interpreters.create() text = self.run_and_capture(orig, script) - interpid, owned = parse_stdout(text) + interpid, = parse_stdout(text) self.assertEqual(interpid, orig) - self.assertTrue(owned) - with self.subTest('not owned'): + with self.subTest('from C-API'): last = 0 for id, *_ in _interpreters.list_all(): last = max(last, id) expected = last + 1 err, text = self.run_temp_external(script) assert err is None, err - interpid, owned = parse_stdout(text) + interpid, = parse_stdout(text) self.assertEqual(interpid, expected) - self.assertFalse(owned) def test_list_all(self): - mainid, _ = _interpreters.get_main() + mainid, *_ = _interpreters.get_main() interpid1 = _interpreters.create() interpid2 = _interpreters.create() interpid3 = _interpreters.create() expected = [ - (mainid, False), - (interpid1, True), - (interpid2, True), - (interpid3, True), + (mainid,), + (interpid1,), + (interpid2,), + (interpid3,), ] with self.subTest('main'): res = _interpreters.list_all() self.assertEqual(res, expected) - with self.subTest('owned'): + with self.subTest('from _interpreters'): text = self.run_and_capture(interpid2, f""" import {_interpreters.__name__} as _interpreters print( @@ -1353,15 +1309,15 @@ def test_list_all(self): res = eval(text) self.assertEqual(res, expected) - with self.subTest('not owned'): + with self.subTest('from C-API'): interpid4 = interpid3 + 1 interpid5 = interpid4 + 1 expected2 = expected + [ - (interpid4, False), - (interpid5, True), + (interpid4,), + (interpid5,), ] expected3 = expected + [ - (interpid5, True), + (interpid5,), ] err, text = self.run_temp_external(f""" import {_interpreters.__name__} as _interpreters @@ -1427,34 +1383,44 @@ def test_create(self): _interpreters.create(orig) def test_destroy(self): - with self.subTest('owned'): + with self.subTest('from _interpreters'): interpid = _interpreters.create() - before = [id for id, _ in _interpreters.list_all()] + before = [id for id, *_ in _interpreters.list_all()] _interpreters.destroy(interpid) - after = [id for id, _ in _interpreters.list_all()] + after = [id for id, *_ in _interpreters.list_all()] self.assertIn(interpid, before) self.assertNotIn(interpid, after) - with self.assertRaises(InterpreterNotFoundError): - _interpreters.is_owned(interpid) + self.assertFalse( + self.interp_exists(interpid)) with self.subTest('main'): - interpid, _ = _interpreters.get_main() + interpid, *_ = _interpreters.get_main() with self.assertRaises(InterpreterError): # It is the current interpreter. _interpreters.destroy(interpid) - with self.subTest('not owned'): + with self.subTest('from C-API'): interpid = _testinternalcapi.create_interpreter() _interpreters.destroy(interpid) - with self.assertRaises(InterpreterNotFoundError): - _interpreters.is_running(interpid) + self.assertFalse( + self.interp_exists(interpid)) def test_get_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + with self.subTest('main'): expected = _interpreters.new_config('legacy') expected.gil = 'own' - interpid, _ = _interpreters.get_main() + interpid, *_ = _interpreters.get_main() config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) @@ -1470,7 +1436,7 @@ def test_get_config(self): config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) - with self.subTest('not owned'): + with self.subTest('from C-API'): orig = _interpreters.new_config('isolated') with self.unmanaged_interpreter(orig) as interpid: config = _interpreters.get_config(interpid) @@ -1478,34 +1444,34 @@ def test_get_config(self): def test_is_running(self): with self.subTest('main'): - interpid, _ = _interpreters.get_main() + interpid, *_ = _interpreters.get_main() running = _interpreters.is_running(interpid) self.assertTrue(running) - with self.subTest('owned (running)'): + with self.subTest('from _interpreters (running)'): interpid = _interpreters.create() with self.running(interpid): running = _interpreters.is_running(interpid) self.assertTrue(running) - with self.subTest('owned (not running)'): + with self.subTest('from _interpreters (not running)'): interpid = _interpreters.create() running = _interpreters.is_running(interpid) self.assertFalse(running) - with self.subTest('not owned (running __main__)'): + with self.subTest('from C-API (running __main__)'): with self.unmanaged_interpreter() as interpid: with self.running_external(interpid, main=True): running = _interpreters.is_running(interpid) self.assertTrue(running) - with self.subTest('not owned (running, but not __main__)'): + with self.subTest('from C-API (running, but not __main__)'): with self.unmanaged_interpreter() as interpid: with self.running_external(interpid, main=False): running = _interpreters.is_running(interpid) self.assertFalse(running) - with self.subTest('not owned (not running)'): + with self.subTest('from C-API (not running)'): with self.unmanaged_interpreter() as interpid: running = _interpreters.is_running(interpid) self.assertFalse(running) @@ -1542,7 +1508,7 @@ def test_exec(self): errdisplay=exc.errdisplay, )) - with self.subTest('not owned'): + with self.subTest('from C-API'): with self.unmanaged_interpreter() as interpid: exc = _interpreters.exec(interpid, 'raise Exception("it worked!")') self.assertIsNot(exc, None) @@ -1570,7 +1536,7 @@ def test_call(self): )) def test_set___main___attrs(self): - with self.subTest('owned'): + with self.subTest('from _interpreters'): interpid = _interpreters.create() before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') before2 = _interpreters.exec(interpid, 'assert ham == 42') @@ -1588,7 +1554,7 @@ def test_set___main___attrs(self): self.assertIs(after2, None) self.assertEqual(after3.type.__name__, 'AssertionError') - with self.subTest('not owned'): + with self.subTest('from C-API'): with self.unmanaged_interpreter() as interpid: _interpreters.set___main___attrs(interpid, {'spam': True}) exc = _interpreters.exec(interpid, 'assert spam is True') diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index bf79a5b6508d24..3f4366ec5663eb 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -509,6 +509,11 @@ def unmanaged_interpreter(self, config='legacy'): except _interpreters.InterpreterNotFoundError: pass + @contextlib.contextmanager + def unmanaged_interpreter_obj(self, config='legacy'): + with self.unmanaged_interpreter(config) as interpid: + yield interpreters.Interpreter(interpid), interpid + @contextlib.contextmanager def capturing(self, script): wrapped, capturing = _captured_script(script, stdout=True, exc=True) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index be571c8cd646ca..240ad46ef4b5f5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -491,6 +491,19 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ +static PyObject * +get_summary(PyInterpreterState *interp) +{ + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + return NULL; + } + PyObject *res = PyTuple_Pack(1, idobj); + Py_DECREF(idobj); + return res; +} + + static PyObject * interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) { @@ -645,7 +658,7 @@ So does an unrecognized ID."); static PyObject * interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *ids, *id; + PyObject *ids; PyInterpreterState *interp; ids = PyList_New(0); @@ -655,14 +668,14 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) interp = PyInterpreterState_Head(); while (interp != NULL) { - id = _PyInterpreterState_GetIDObject(interp); - if (id == NULL) { + PyObject *item = get_summary(interp); + if (item == NULL) { Py_DECREF(ids); return NULL; } // insert at front of list - int res = PyList_Insert(ids, 0, id); - Py_DECREF(id); + int res = PyList_Insert(ids, 0, item); + Py_DECREF(item); if (res < 0) { Py_DECREF(ids); return NULL; @@ -675,7 +688,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(list_all_doc, -"list_all() -> [ID]\n\ +"list_all() -> [(ID,)]\n\ \n\ Return a list containing the ID of every existing interpreter."); @@ -687,11 +700,11 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } - return _PyInterpreterState_GetIDObject(interp); + return get_summary(interp); } PyDoc_STRVAR(get_current_doc, -"get_current() -> ID\n\ +"get_current() -> (ID,)\n\ \n\ Return the ID of current interpreter."); @@ -699,13 +712,12 @@ Return the ID of current interpreter."); static PyObject * interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) { - // Currently, 0 is always the main interpreter. - int64_t id = 0; - return PyLong_FromLongLong(id); + PyInterpreterState *interp = _PyInterpreterState_Main(); + return get_summary(interp); } PyDoc_STRVAR(get_main_doc, -"get_main() -> ID\n\ +"get_main() -> (ID,)\n\ \n\ Return the ID of main interpreter."); From b1f96d2b4622b51c7d11844d33fa2de68507a85a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 18:30:53 -0600 Subject: [PATCH 15/28] Fix clean_up_interpreters(). --- Lib/test/test_interpreters/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 3f4366ec5663eb..fd38c72168b8d8 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -329,7 +329,7 @@ def clean_up_interpreters(): continue try: interp.close() - except RuntimeError: + except _interpreters.InterpreterError: pass # already destroyed From cd616438bb8e8bc27865080325a46cabdde4f484 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 18:33:06 -0600 Subject: [PATCH 16/28] Fix test_capi.test_misc. --- Lib/test/test_capi/test_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 2f2bf03749f834..db5dfa00d42f00 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2579,7 +2579,7 @@ def test_linked_lifecycle_does_not_exist(self): def test_linked_lifecycle_initial(self): is_linked = _testinternalcapi.interpreter_refcount_linked - get_refcount = _testinternalcapi.get_interpreter_refcount + get_refcount, _, _ = self.get_refcount_helpers() # A new interpreter will start out not linked, with a refcount of 0. interpid = self.new_interpreter() From 7de523dedb759087ab2e77e9b2492bc990767f38 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 18:49:02 -0600 Subject: [PATCH 17/28] external/unmanaged -> from_capi --- Lib/test/test_interpreters/test_api.py | 60 +++++++++++++------------- Lib/test/test_interpreters/utils.py | 14 +++--- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 484e16ebb73133..462fb1d3dbf9c6 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -169,7 +169,7 @@ def test_created_with_capi(self): for id, *_ in _interpreters.list_all(): last = max(last, id) expected = _testinternalcapi.next_interpreter_id() - err, text = self.run_temp_external(f""" + err, text = self.run_temp_from_capi(f""" import {interpreters.__name__} as interpreters interp = interpreters.get_current() print(interp.id) @@ -236,7 +236,7 @@ def test_created_with_capi(self): (interpid5,), ] expected2 = expected[:-2] - err, text = self.run_temp_external(f""" + err, text = self.run_temp_from_capi(f""" import {interpreters.__name__} as interpreters interp = interpreters.create() print( @@ -402,20 +402,20 @@ def resolve_results(err, text): raise Exception(repr(text)) with self.subTest('running __main__ (from self)'): - with self.unmanaged_interpreter() as interpid: - err, text = self.run_external(interpid, script, main=True) + with self.interpreter_from_capi() as interpid: + err, text = self.run_from_capi(interpid, script, main=True) running = resolve_results(err, text) self.assertTrue(running) with self.subTest('running, but not __main__ (from self)'): - err, text = self.run_temp_external(script) + err, text = self.run_temp_from_capi(script) running = resolve_results(err, text) self.assertFalse(running) with self.subTest('running __main__ (from other)'): - with self.unmanaged_interpreter_obj() as (interp, interpid): + with self.interpreter_obj_from_capi() as (interp, interpid): before = interp.is_running() - with self.running_external(interpid, main=True): + with self.running_from_capi(interpid, main=True): during = interp.is_running() after = interp.is_running() self.assertFalse(before) @@ -423,9 +423,9 @@ def resolve_results(err, text): self.assertFalse(after) with self.subTest('running, but not __main__ (from other)'): - with self.unmanaged_interpreter_obj() as (interp, interpid): + with self.interpreter_obj_from_capi() as (interp, interpid): before = interp.is_running() - with self.running_external(interpid, main=False): + with self.running_from_capi(interpid, main=False): during = interp.is_running() after = interp.is_running() self.assertFalse(before) @@ -433,7 +433,7 @@ def resolve_results(err, text): self.assertFalse(after) with self.subTest('not running (from other)'): - with self.unmanaged_interpreter_obj() as (interp, _): + with self.interpreter_obj_from_capi() as (interp, _): running = interp.is_running() self.assertFalse(running) @@ -573,17 +573,17 @@ def check_results(err, text): self.assertEqual(text, '') with self.subTest('running __main__ (from self)'): - with self.unmanaged_interpreter() as interpid: - err, text = self.run_external(interpid, script, main=True) + with self.interpreter_from_capi() as interpid: + err, text = self.run_from_capi(interpid, script, main=True) check_results(err, text) with self.subTest('running, but not __main__ (from self)'): - err, text = self.run_temp_external(script) + err, text = self.run_temp_from_capi(script) check_results(err, text) with self.subTest('running __main__ (from other)'): - with self.unmanaged_interpreter_obj() as (interp, interpid): - with self.running_external(interpid, main=True): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=True): with self.assertRaisesRegex(InterpreterError, 'running'): interp.close() # Make sure it wssn't closed. @@ -595,15 +595,15 @@ def check_results(err, text): return with self.subTest('running, but not __main__ (from other)'): - with self.unmanaged_interpreter_obj() as (interp, interpid): - with self.running_external(interpid, main=False): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=False): with self.assertRaisesRegex(InterpreterError, 'not managed'): interp.close() # Make sure it wssn't closed. self.assertFalse(interp.is_running()) with self.subTest('not running (from other)'): - with self.unmanaged_interpreter_obj() as (interp, _): + with self.interpreter_obj_from_capi() as (interp, _): with self.assertRaisesRegex(InterpreterError, 'not managed'): interp.close() # Make sure it wssn't closed. @@ -676,7 +676,7 @@ def test_running(self): @requires__testinternalcapi def test_created_with_capi(self): - with self.unmanaged_interpreter() as interpid: + with self.interpreter_from_capi() as interpid: interp = interpreters.Interpreter(interpid) interp.prepare_main({'spam': True}) rc = _testinternalcapi.exec_interpreter(interpid, @@ -835,7 +835,7 @@ def task(): self.assertEqual(os.read(r_interp, 1), FINISHED) def test_created_with_capi(self): - with self.unmanaged_interpreter_obj() as (interp, _): + with self.interpreter_obj_from_capi() as (interp, _): with self.assertRaisesRegex(ExecutionFailed, 'it worked'): interp.exec('raise Exception("it worked!")') @@ -1278,7 +1278,7 @@ def parse_stdout(text): for id, *_ in _interpreters.list_all(): last = max(last, id) expected = last + 1 - err, text = self.run_temp_external(script) + err, text = self.run_temp_from_capi(script) assert err is None, err interpid, = parse_stdout(text) self.assertEqual(interpid, expected) @@ -1319,7 +1319,7 @@ def test_list_all(self): expected3 = expected + [ (interpid5,), ] - err, text = self.run_temp_external(f""" + err, text = self.run_temp_from_capi(f""" import {_interpreters.__name__} as _interpreters _interpreters.create() print( @@ -1438,7 +1438,7 @@ def test_get_config(self): with self.subTest('from C-API'): orig = _interpreters.new_config('isolated') - with self.unmanaged_interpreter(orig) as interpid: + with self.interpreter_from_capi(orig) as interpid: config = _interpreters.get_config(interpid) self.assert_ns_equal(config, orig) @@ -1460,19 +1460,19 @@ def test_is_running(self): self.assertFalse(running) with self.subTest('from C-API (running __main__)'): - with self.unmanaged_interpreter() as interpid: - with self.running_external(interpid, main=True): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=True): running = _interpreters.is_running(interpid) self.assertTrue(running) with self.subTest('from C-API (running, but not __main__)'): - with self.unmanaged_interpreter() as interpid: - with self.running_external(interpid, main=False): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=False): running = _interpreters.is_running(interpid) self.assertFalse(running) with self.subTest('from C-API (not running)'): - with self.unmanaged_interpreter() as interpid: + with self.interpreter_from_capi() as interpid: running = _interpreters.is_running(interpid) self.assertFalse(running) @@ -1509,7 +1509,7 @@ def test_exec(self): )) with self.subTest('from C-API'): - with self.unmanaged_interpreter() as interpid: + with self.interpreter_from_capi() as interpid: exc = _interpreters.exec(interpid, 'raise Exception("it worked!")') self.assertIsNot(exc, None) self.assertEqual(exc.msg, 'it worked!') @@ -1555,7 +1555,7 @@ def test_set___main___attrs(self): self.assertEqual(after3.type.__name__, 'AssertionError') with self.subTest('from C-API'): - with self.unmanaged_interpreter() as interpid: + with self.interpreter_from_capi() as interpid: _interpreters.set___main___attrs(interpid, {'spam': True}) exc = _interpreters.exec(interpid, 'assert spam is True') self.assertIsNone(exc) diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index fd38c72168b8d8..d4ee2e1562f441 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -497,7 +497,7 @@ def run_and_capture(self, interp, script): @requires__testinternalcapi @contextlib.contextmanager - def unmanaged_interpreter(self, config='legacy'): + def interpreter_from_capi(self, config='legacy'): if isinstance(config, str): config = _interpreters.new_config(config) interpid = _testinternalcapi.create_interpreter() @@ -510,8 +510,8 @@ def unmanaged_interpreter(self, config='legacy'): pass @contextlib.contextmanager - def unmanaged_interpreter_obj(self, config='legacy'): - with self.unmanaged_interpreter(config) as interpid: + def interpreter_obj_from_capi(self, config='legacy'): + with self.interpreter_from_capi(config) as interpid: yield interpreters.Interpreter(interpid), interpid @contextlib.contextmanager @@ -522,7 +522,7 @@ def capturing(self, script): yield wrapped, capturing.final(force=True) @requires__testinternalcapi - def run_external(self, interpid, script, *, main=False): + def run_from_capi(self, interpid, script, *, main=False): with self.capturing(script) as (wrapped, results): rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main) assert rc == 0, rc @@ -602,9 +602,9 @@ def exec_interp(script): @requires__testinternalcapi @contextlib.contextmanager - def running_external(self, interpid, *, main=False): + def running_from_capi(self, interpid, *, main=False): def run_interp(script): - err, text = self.run_external(interpid, script, main=main) + err, text = self.run_from_capi(interpid, script, main=main) assert err is None, err assert text == '', repr(text) def exec_interp(script): @@ -614,7 +614,7 @@ def exec_interp(script): yield @requires__testinternalcapi - def run_temp_external(self, script, config='legacy'): + def run_temp_from_capi(self, script, config='legacy'): if isinstance(config, str): config = _interpreters.new_config(config) with self.capturing(script) as (wrapped, results): From 11d38aba5cf5433f769d10e4de0280d6dda1144c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 19:21:16 -0600 Subject: [PATCH 18/28] Add a missing decorator. --- Lib/test/test_interpreters/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 462fb1d3dbf9c6..b4b1b69cdc1edc 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1382,6 +1382,7 @@ def test_create(self): with self.assertRaises(ValueError): _interpreters.create(orig) + @requires__testinternalcapi def test_destroy(self): with self.subTest('from _interpreters'): interpid = _interpreters.create() From 1cf31b6501c894e018434d8488d40f23767a956b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Apr 2024 20:21:54 -0600 Subject: [PATCH 19/28] Fix other tests. --- Lib/test/test__xxinterpchannels.py | 111 ++++----- Lib/test/test__xxsubinterpreters.py | 338 +++++++++++++++------------- Lib/test/test_capi/test_misc.py | 4 +- 3 files changed, 234 insertions(+), 219 deletions(-) diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index c5d29bd2dd911f..3db0cb7e6e1d49 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -9,7 +9,7 @@ from test.support import import_helper from test.test__xxsubinterpreters import ( - interpreters, + _interpreters, _run_output, clean_up_interpreters, ) @@ -49,14 +49,15 @@ def run_interp(id, source, **shared): def _run_interp(id, source, shared, _mainns={}): source = dedent(source) - main = interpreters.get_main() + main, *_ = _interpreters.get_main() if main == id: - if interpreters.get_current() != main: + cur, *_ = _interpreters.get_current() + if cur != main: raise RuntimeError # XXX Run a func? exec(source, _mainns) else: - interpreters.run_string(id, source, shared) + _interpreters.run_string(id, source, shared) class Interpreter(namedtuple('Interpreter', 'name id')): @@ -71,7 +72,7 @@ def from_raw(cls, raw): raise NotImplementedError def __new__(cls, name=None, id=None): - main = interpreters.get_main() + main, *_ = _interpreters.get_main() if id == main: if not name: name = 'main' @@ -89,7 +90,7 @@ def __new__(cls, name=None, id=None): name = 'main' id = main else: - id = interpreters.create() + id = _interpreters.create() self = super().__new__(cls, name, id) return self @@ -370,7 +371,7 @@ def test_sequential_ids(self): self.assertEqual(set(after) - set(before), {id1, id2, id3}) def test_ids_global(self): - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(""" import _xxinterpchannels as _channels cid = _channels.create() @@ -378,7 +379,7 @@ def test_ids_global(self): """)) cid1 = int(out.strip()) - id2 = interpreters.create() + id2 = _interpreters.create() out = _run_output(id2, dedent(""" import _xxinterpchannels as _channels cid = _channels.create() @@ -390,7 +391,7 @@ def test_ids_global(self): def test_channel_list_interpreters_none(self): """Test listing interpreters for a channel with no associations.""" - # Test for channel with no associated interpreters. + # Test for channel with no associated _interpreters. cid = channels.create() send_interps = channels.list_interpreters(cid, send=True) recv_interps = channels.list_interpreters(cid, send=False) @@ -398,8 +399,8 @@ def test_channel_list_interpreters_none(self): self.assertEqual(recv_interps, []) def test_channel_list_interpreters_basic(self): - """Test basic listing channel interpreters.""" - interp0 = interpreters.get_main() + """Test basic listing channel _interpreters.""" + interp0, *_ = _interpreters.get_main() cid = channels.create() channels.send(cid, "send", blocking=False) # Test for a channel that has one end associated to an interpreter. @@ -408,7 +409,7 @@ def test_channel_list_interpreters_basic(self): self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, []) - interp1 = interpreters.create() + interp1 = _interpreters.create() _run_output(interp1, dedent(f""" import _xxinterpchannels as _channels obj = _channels.recv({cid}) @@ -421,10 +422,10 @@ def test_channel_list_interpreters_basic(self): def test_channel_list_interpreters_multiple(self): """Test listing interpreters for a channel with many associations.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + interp2 = _interpreters.create() + interp3 = _interpreters.create() cid = channels.create() channels.send(cid, "send", blocking=False) @@ -447,8 +448,8 @@ def test_channel_list_interpreters_multiple(self): def test_channel_list_interpreters_destroyed(self): """Test listing channel interpreters with a destroyed interpreter.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() cid = channels.create() channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" @@ -461,7 +462,7 @@ def test_channel_list_interpreters_destroyed(self): self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, [interp1]) - interpreters.destroy(interp1) + _interpreters.destroy(interp1) # Destroyed interpreter should not be listed. send_interps = channels.list_interpreters(cid, send=True) recv_interps = channels.list_interpreters(cid, send=False) @@ -472,9 +473,9 @@ def test_channel_list_interpreters_released(self): """Test listing channel interpreters with a released channel.""" # Set up one channel with main interpreter on the send end and two # subinterpreters on the receive end. - interp0 = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + interp2 = _interpreters.create() cid = channels.create() channels.send(cid, "data", blocking=False) _run_output(interp1, dedent(f""" @@ -494,7 +495,7 @@ def test_channel_list_interpreters_released(self): # Release the main interpreter from the send end. channels.release(cid, send=True) - # Send end should have no associated interpreters. + # Send end should have no associated _interpreters. send_interps = channels.list_interpreters(cid, send=True) recv_interps = channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 0) @@ -513,8 +514,8 @@ def test_channel_list_interpreters_released(self): def test_channel_list_interpreters_closed(self): """Test listing channel interpreters with a closed channel.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() cid = channels.create() # Put something in the channel so that it's not empty. channels.send(cid, "send", blocking=False) @@ -535,8 +536,8 @@ def test_channel_list_interpreters_closed(self): def test_channel_list_interpreters_closed_send_end(self): """Test listing channel interpreters with a channel's send end closed.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() cid = channels.create() # Put something in the channel so that it's not empty. channels.send(cid, "send", blocking=False) @@ -589,9 +590,9 @@ def test_allowed_types(self): def test_run_string_arg_unresolved(self): cid = channels.create() - interp = interpreters.create() + interp = _interpreters.create() - interpreters.set___main___attrs(interp, dict(cid=cid.send)) + _interpreters.set___main___attrs(interp, dict(cid=cid.send)) out = _run_output(interp, dedent(""" import _xxinterpchannels as _channels print(cid.end) @@ -609,7 +610,7 @@ def test_run_string_arg_unresolved(self): def test_run_string_arg_resolved(self): cid = channels.create() cid = channels._channel_id(cid, _resolve=True) - interp = interpreters.create() + interp = _interpreters.create() out = _run_output(interp, dedent(""" import _xxinterpchannels as _channels @@ -635,7 +636,7 @@ def test_send_recv_main(self): self.assertIsNot(obj, orig) def test_send_recv_same_interpreter(self): - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(""" import _xxinterpchannels as _channels cid = _channels.create() @@ -648,7 +649,7 @@ def test_send_recv_same_interpreter(self): def test_send_recv_different_interpreters(self): cid = channels.create() - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(f""" import _xxinterpchannels as _channels _channels.send({cid}, b'spam', blocking=False) @@ -674,7 +675,7 @@ def f(): def test_send_recv_different_interpreters_and_threads(self): cid = channels.create() - id1 = interpreters.create() + id1 = _interpreters.create() out = None def f(): @@ -737,12 +738,12 @@ def test_recv_default(self): def test_recv_sending_interp_destroyed(self): with self.subTest('closed'): cid1 = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" import _xxinterpchannels as _channels _channels.send({cid1}, b'spam', blocking=False) """)) - interpreters.destroy(interp) + _interpreters.destroy(interp) with self.assertRaisesRegex(RuntimeError, f'channel {cid1} is closed'): @@ -750,13 +751,13 @@ def test_recv_sending_interp_destroyed(self): del cid1 with self.subTest('still open'): cid2 = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" import _xxinterpchannels as _channels _channels.send({cid2}, b'spam', blocking=False) """)) channels.send(cid2, b'eggs', blocking=False) - interpreters.destroy(interp) + _interpreters.destroy(interp) channels.recv(cid2) with self.assertRaisesRegex(RuntimeError, @@ -1010,24 +1011,24 @@ def test_close_single_user(self): def test_close_multiple_users(self): cid = channels.create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" + id1 = _interpreters.create() + id2 = _interpreters.create() + _interpreters.run_string(id1, dedent(f""" import _xxinterpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) - interpreters.run_string(id2, dedent(f""" + _interpreters.run_string(id2, dedent(f""" import _xxinterpchannels as _channels _channels.recv({cid}) """)) channels.close(cid) - excsnap = interpreters.run_string(id1, dedent(f""" + excsnap = _interpreters.run_string(id1, dedent(f""" _channels.send({cid}, b'spam') """)) self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') - excsnap = interpreters.run_string(id2, dedent(f""" + excsnap = _interpreters.run_string(id2, dedent(f""" _channels.send({cid}, b'spam') """)) self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') @@ -1154,8 +1155,8 @@ def test_close_never_used(self): def test_close_by_unassociated_interp(self): cid = channels.create() channels.send(cid, b'spam', blocking=False) - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" import _xxinterpchannels as _channels _channels.close({cid}, force=True) """)) @@ -1251,9 +1252,9 @@ def test_single_user(self): def test_multiple_users(self): cid = channels.create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" + id1 = _interpreters.create() + id2 = _interpreters.create() + _interpreters.run_string(id1, dedent(f""" import _xxinterpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) @@ -1263,7 +1264,7 @@ def test_multiple_users(self): _channels.release({cid}) print(repr(obj)) """)) - interpreters.run_string(id1, dedent(f""" + _interpreters.run_string(id1, dedent(f""" _channels.release({cid}) """)) @@ -1310,8 +1311,8 @@ def test_never_used(self): def test_by_unassociated_interp(self): cid = channels.create() channels.send(cid, b'spam', blocking=False) - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" import _xxinterpchannels as _channels _channels.release({cid}) """)) @@ -1325,8 +1326,8 @@ def test_by_unassociated_interp(self): def test_close_if_unassociated(self): # XXX Something's not right with this test... cid = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" import _xxinterpchannels as _channels obj = _channels.send({cid}, b'spam', blocking=False) _channels.release({cid}) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 841077adbb0f16..c8c964f642f1cf 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -13,7 +13,7 @@ from test.support import script_helper -interpreters = import_helper.import_module('_xxsubinterpreters') +_interpreters = import_helper.import_module('_xxsubinterpreters') _testinternalcapi = import_helper.import_module('_testinternalcapi') from _xxsubinterpreters import InterpreterNotFoundError @@ -36,7 +36,7 @@ def _captured_script(script): def _run_output(interp, request): script, rpipe = _captured_script(request) with rpipe: - interpreters.run_string(interp, script) + _interpreters.run_string(interp, script) return rpipe.read() @@ -47,7 +47,7 @@ def _wait_for_interp_to_run(interp, timeout=None): if timeout is None: timeout = support.SHORT_TIMEOUT for _ in support.sleeping_retry(timeout, error=False): - if interpreters.is_running(interp): + if _interpreters.is_running(interp): break else: raise RuntimeError('interp is not running') @@ -57,7 +57,7 @@ def _wait_for_interp_to_run(interp, timeout=None): def _running(interp): r, w = os.pipe() def run(): - interpreters.run_string(interp, dedent(f""" + _interpreters.run_string(interp, dedent(f""" # wait for "signal" with open({r}, encoding="utf-8") as rpipe: rpipe.read() @@ -75,12 +75,12 @@ def run(): def clean_up_interpreters(): - for id in interpreters.list_all(): + for id, *_ in _interpreters.list_all(): if id == 0: # main continue try: - interpreters.destroy(id) - except interpreters.InterpreterError: + _interpreters.destroy(id) + except _interpreters.InterpreterError: pass # already destroyed @@ -112,7 +112,7 @@ def test_default_shareables(self): for obj in shareables: with self.subTest(obj): self.assertTrue( - interpreters.is_shareable(obj)) + _interpreters.is_shareable(obj)) def test_not_shareable(self): class Cheese: @@ -141,7 +141,7 @@ class SubBytes(bytes): for obj in not_shareables: with self.subTest(repr(obj)): self.assertFalse( - interpreters.is_shareable(obj)) + _interpreters.is_shareable(obj)) class ShareableTypeTests(unittest.TestCase): @@ -230,7 +230,7 @@ class ModuleTests(TestBase): def test_import_in_interpreter(self): _run_output( - interpreters.create(), + _interpreters.create(), 'import _xxsubinterpreters as _interpreters', ) @@ -241,45 +241,45 @@ def test_import_in_interpreter(self): class ListAllTests(TestBase): def test_initial(self): - main = interpreters.get_main() - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main]) def test_after_creating(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + first = _interpreters.create() + second = _interpreters.create() + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main, first, second]) def test_after_destroying(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - interpreters.destroy(first) - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + first = _interpreters.create() + second = _interpreters.create() + _interpreters.destroy(first) + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main, second]) class GetCurrentTests(TestBase): def test_main(self): - main = interpreters.get_main() - cur = interpreters.get_current() + main, *_ = _interpreters.get_main() + cur, *_ = _interpreters.get_current() self.assertEqual(cur, main) self.assertIsInstance(cur, int) def test_subinterpreter(self): - main = interpreters.get_main() - interp = interpreters.create() + main, *_ = _interpreters.get_main() + interp = _interpreters.create() out = _run_output(interp, dedent(""" import _xxsubinterpreters as _interpreters - cur = _interpreters.get_current() + cur, *_ = _interpreters.get_current() print(cur) assert isinstance(cur, int) """)) cur = int(out.strip()) - _, expected = interpreters.list_all() + _, expected = [id for id, *_ in _interpreters.list_all()] self.assertEqual(cur, expected) self.assertNotEqual(cur, main) @@ -287,17 +287,17 @@ def test_subinterpreter(self): class GetMainTests(TestBase): def test_from_main(self): - [expected] = interpreters.list_all() - main = interpreters.get_main() + [expected] = [id for id, *_ in _interpreters.list_all()] + main, *_ = _interpreters.get_main() self.assertEqual(main, expected) self.assertIsInstance(main, int) def test_from_subinterpreter(self): - [expected] = interpreters.list_all() - interp = interpreters.create() + [expected] = [id for id, *_ in _interpreters.list_all()] + interp = _interpreters.create() out = _run_output(interp, dedent(""" import _xxsubinterpreters as _interpreters - main = _interpreters.get_main() + main, *_ = _interpreters.get_main() print(main) assert isinstance(main, int) """)) @@ -308,20 +308,20 @@ def test_from_subinterpreter(self): class IsRunningTests(TestBase): def test_main(self): - main = interpreters.get_main() - self.assertTrue(interpreters.is_running(main)) + main, *_ = _interpreters.get_main() + self.assertTrue(_interpreters.is_running(main)) @unittest.skip('Fails on FreeBSD') def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interpreters.is_running(interp)) + interp = _interpreters.create() + self.assertFalse(_interpreters.is_running(interp)) with _running(interp): - self.assertTrue(interpreters.is_running(interp)) - self.assertFalse(interpreters.is_running(interp)) + self.assertTrue(_interpreters.is_running(interp)) + self.assertFalse(_interpreters.is_running(interp)) def test_from_subinterpreter(self): - interp = interpreters.create() + interp = _interpreters.create() out = _run_output(interp, dedent(f""" import _xxsubinterpreters as _interpreters if _interpreters.is_running({interp}): @@ -332,34 +332,35 @@ def test_from_subinterpreter(self): self.assertEqual(out.strip(), 'True') def test_already_destroyed(self): - interp = interpreters.create() - interpreters.destroy(interp) + interp = _interpreters.create() + _interpreters.destroy(interp) with self.assertRaises(InterpreterNotFoundError): - interpreters.is_running(interp) + _interpreters.is_running(interp) def test_does_not_exist(self): with self.assertRaises(InterpreterNotFoundError): - interpreters.is_running(1_000_000) + _interpreters.is_running(1_000_000) def test_bad_id(self): with self.assertRaises(ValueError): - interpreters.is_running(-1) + _interpreters.is_running(-1) class CreateTests(TestBase): def test_in_main(self): - id = interpreters.create() + id = _interpreters.create() self.assertIsInstance(id, int) - self.assertIn(id, interpreters.list_all()) + after = [id for id, *_ in _interpreters.list_all()] + self.assertIn(id, after) @unittest.skip('enable this test when working on pystate.c') def test_unique_id(self): seen = set() for _ in range(100): - id = interpreters.create() - interpreters.destroy(id) + id = _interpreters.create() + _interpreters.destroy(id) seen.add(id) self.assertEqual(len(seen), 100) @@ -369,7 +370,7 @@ def test_in_thread(self): id = None def f(): nonlocal id - id = interpreters.create() + id = _interpreters.create() lock.acquire() lock.release() @@ -377,11 +378,12 @@ def f(): with lock: t.start() t.join() - self.assertIn(id, interpreters.list_all()) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertIn(id, after) def test_in_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() out = _run_output(id1, dedent(""" import _xxsubinterpreters as _interpreters id = _interpreters.create() @@ -390,11 +392,12 @@ def test_in_subinterpreter(self): """)) id2 = int(out.strip()) - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1, id2}) def test_in_threaded_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() id2 = None def f(): nonlocal id2 @@ -409,144 +412,155 @@ def f(): t.start() t.join() - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1, id2}) def test_after_destroy_all(self): - before = set(interpreters.list_all()) + before = set(id for id, *_ in _interpreters.list_all()) # Create 3 subinterpreters. ids = [] for _ in range(3): - id = interpreters.create() + id = _interpreters.create() ids.append(id) # Now destroy them. for id in ids: - interpreters.destroy(id) + _interpreters.destroy(id) # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id}) + id = _interpreters.create() + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, before | {id}) def test_after_destroy_some(self): - before = set(interpreters.list_all()) + before = set(id for id, *_ in _interpreters.list_all()) # Create 3 subinterpreters. - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() + id1 = _interpreters.create() + id2 = _interpreters.create() + id3 = _interpreters.create() # Now destroy 2 of them. - interpreters.destroy(id1) - interpreters.destroy(id3) + _interpreters.destroy(id1) + _interpreters.destroy(id3) # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id, id2}) + id = _interpreters.create() + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, before | {id, id2}) class DestroyTests(TestBase): def test_one(self): - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() - self.assertIn(id2, interpreters.list_all()) - interpreters.destroy(id2) - self.assertNotIn(id2, interpreters.list_all()) - self.assertIn(id1, interpreters.list_all()) - self.assertIn(id3, interpreters.list_all()) + id1 = _interpreters.create() + id2 = _interpreters.create() + id3 = _interpreters.create() + before = set(id for id, *_ in _interpreters.list_all()) + self.assertIn(id2, before) + + _interpreters.destroy(id2) + + after = set(id for id, *_ in _interpreters.list_all()) + self.assertNotIn(id2, after) + self.assertIn(id1, after) + self.assertIn(id3, after) def test_all(self): - before = set(interpreters.list_all()) + initial = set(id for id, *_ in _interpreters.list_all()) ids = set() for _ in range(3): - id = interpreters.create() + id = _interpreters.create() ids.add(id) - self.assertEqual(set(interpreters.list_all()), before | ids) + before = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(before, initial | ids) for id in ids: - interpreters.destroy(id) - self.assertEqual(set(interpreters.list_all()), before) + _interpreters.destroy(id) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, initial) def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(interpreters.InterpreterError): - interpreters.destroy(main) + main, = [id for id, *_ in _interpreters.list_all()] + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.destroy(main) def f(): - with self.assertRaises(interpreters.InterpreterError): - interpreters.destroy(main) + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.destroy(main) t = threading.Thread(target=f) t.start() t.join() def test_already_destroyed(self): - id = interpreters.create() - interpreters.destroy(id) + id = _interpreters.create() + _interpreters.destroy(id) with self.assertRaises(InterpreterNotFoundError): - interpreters.destroy(id) + _interpreters.destroy(id) def test_does_not_exist(self): with self.assertRaises(InterpreterNotFoundError): - interpreters.destroy(1_000_000) + _interpreters.destroy(1_000_000) def test_bad_id(self): with self.assertRaises(ValueError): - interpreters.destroy(-1) + _interpreters.destroy(-1) def test_from_current(self): - main, = interpreters.list_all() - id = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id = _interpreters.create() script = dedent(f""" import _xxsubinterpreters as _interpreters try: _interpreters.destroy({id}) - except interpreters.InterpreterError: + except _interpreters.InterpreterError: pass """) - interpreters.run_string(id, script) - self.assertEqual(set(interpreters.list_all()), {main, id}) + _interpreters.run_string(id, script) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id}) def test_from_sibling(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() + id2 = _interpreters.create() script = dedent(f""" import _xxsubinterpreters as _interpreters _interpreters.destroy({id2}) """) - interpreters.run_string(id1, script) + _interpreters.run_string(id1, script) - self.assertEqual(set(interpreters.list_all()), {main, id1}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1}) def test_from_other_thread(self): - id = interpreters.create() + id = _interpreters.create() def f(): - interpreters.destroy(id) + _interpreters.destroy(id) t = threading.Thread(target=f) t.start() t.join() def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + interp = _interpreters.create() with _running(interp): - self.assertTrue(interpreters.is_running(interp), + self.assertTrue(_interpreters.is_running(interp), msg=f"Interp {interp} should be running before destruction.") - with self.assertRaises(interpreters.InterpreterError, + with self.assertRaises(_interpreters.InterpreterError, msg=f"Should not be able to destroy interp {interp} while it's still running."): - interpreters.destroy(interp) - self.assertTrue(interpreters.is_running(interp)) + _interpreters.destroy(interp) + self.assertTrue(_interpreters.is_running(interp)) class RunStringTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def test_success(self): script, file = _captured_script('print("it worked!", end="")') with file: - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) out = file.read() self.assertEqual(out, 'it worked!') @@ -555,7 +569,7 @@ def test_in_thread(self): script, file = _captured_script('print("it worked!", end="")') with file: def f(): - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) t = threading.Thread(target=f) t.start() @@ -565,7 +579,7 @@ def f(): self.assertEqual(out, 'it worked!') def test_create_thread(self): - subinterp = interpreters.create() + subinterp = _interpreters.create() script, file = _captured_script(""" import threading def f(): @@ -576,7 +590,7 @@ def f(): t.join() """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, 'it worked!') @@ -584,7 +598,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create('isolated') + subinterp = _interpreters.create('isolated') script, file = _captured_script(f""" import threading def f(): @@ -598,13 +612,13 @@ def f(): print('{expected}', end='') """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create('legacy') + subinterp = _interpreters.create('legacy') script, file = _captured_script(""" import threading def f(): @@ -615,13 +629,13 @@ def f(): t.join() """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, 'it worked!') def test_shareable_types(self): - interp = interpreters.create() + interp = _interpreters.create() objects = [ None, 'spam', @@ -630,15 +644,15 @@ def test_shareable_types(self): ] for obj in objects: with self.subTest(obj): - interpreters.set___main___attrs(interp, dict(obj=obj)) - interpreters.run_string( + _interpreters.set___main___attrs(interp, dict(obj=obj)) + _interpreters.run_string( interp, f'assert(obj == {obj!r})', ) def test_os_exec(self): expected = 'spam spam spam spam spam' - subinterp = interpreters.create() + subinterp = _interpreters.create() script, file = _captured_script(f""" import os, sys try: @@ -647,7 +661,7 @@ def test_os_exec(self): print('{expected}', end='') """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, expected) @@ -668,7 +682,7 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) file.seek(0) content = file.read() @@ -676,31 +690,31 @@ def test_fork(self): def test_already_running(self): with _running(self.id): - with self.assertRaises(interpreters.InterpreterError): - interpreters.run_string(self.id, 'print("spam")') + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.run_string(self.id, 'print("spam")') def test_does_not_exist(self): id = 0 - while id in interpreters.list_all(): + while id in set(id for id, *_ in _interpreters.list_all()): id += 1 with self.assertRaises(InterpreterNotFoundError): - interpreters.run_string(id, 'print("spam")') + _interpreters.run_string(id, 'print("spam")') def test_error_id(self): with self.assertRaises(ValueError): - interpreters.run_string(-1, 'print("spam")') + _interpreters.run_string(-1, 'print("spam")') def test_bad_id(self): with self.assertRaises(TypeError): - interpreters.run_string('spam', 'print("spam")') + _interpreters.run_string('spam', 'print("spam")') def test_bad_script(self): with self.assertRaises(TypeError): - interpreters.run_string(self.id, 10) + _interpreters.run_string(self.id, 10) def test_bytes_for_script(self): with self.assertRaises(TypeError): - interpreters.run_string(self.id, b'print("spam")') + _interpreters.run_string(self.id, b'print("spam")') def test_with_shared(self): r, w = os.pipe() @@ -721,8 +735,8 @@ def test_with_shared(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -732,7 +746,7 @@ def test_with_shared(self): self.assertIsNone(ns['cheddar']) def test_shared_overwrites(self): - interpreters.run_string(self.id, dedent(""" + _interpreters.run_string(self.id, dedent(""" spam = 'eggs' ns1 = dict(vars()) del ns1['__builtins__'] @@ -743,8 +757,8 @@ def test_shared_overwrites(self): ns2 = dict(vars()) del ns2['__builtins__'] """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) r, w = os.pipe() script = dedent(f""" @@ -754,7 +768,7 @@ def test_shared_overwrites(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -775,8 +789,8 @@ def test_shared_overwrites_default_vars(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -784,7 +798,7 @@ def test_shared_overwrites_default_vars(self): def test_main_reused(self): r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" spam = True ns = dict(vars()) @@ -798,7 +812,7 @@ def test_main_reused(self): ns1 = pickle.load(chan) r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" eggs = False ns = dict(vars()) @@ -827,7 +841,7 @@ def test_execution_namespace_is_main(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -872,13 +886,13 @@ class RunFailedTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def add_module(self, modname, text): import tempfile tempdir = tempfile.mkdtemp() self.addCleanup(lambda: os_helper.rmtree(tempdir)) - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" import sys sys.path.insert(0, {tempdir!r}) """)) @@ -900,11 +914,11 @@ class NeverError(Exception): pass raise NeverError # never raised """).format(dedent(text)) if fails: - err = interpreters.run_string(self.id, script) + err = _interpreters.run_string(self.id, script) self.assertIsNot(err, None) return err else: - err = interpreters.run_string(self.id, script) + err = _interpreters.run_string(self.id, script) self.assertIs(err, None) return None except: @@ -1029,7 +1043,7 @@ class RunFuncTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def test_success(self): r, w = os.pipe() @@ -1039,8 +1053,8 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, script) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1056,8 +1070,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') def f(): - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, script) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) t = threading.Thread(target=f) t.start() t.join() @@ -1077,8 +1091,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') code = script.__code__ - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, code) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, code) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1091,7 +1105,7 @@ def script(): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) # XXX This hasn't been fixed yet. @unittest.expectedFailure @@ -1099,38 +1113,38 @@ def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) def test_args(self): with self.subTest('args'): def script(a, b=0): assert a == b with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('*args'): def script(*args): assert not args with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('**kwargs'): def script(**kwargs): assert not kwargs with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('kwonly'): def script(*, spam=True): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('posonly'): def script(spam, /): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) if __name__ == '__main__': diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index db5dfa00d42f00..a76b866f25f804 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2403,7 +2403,7 @@ def check(config): continue if match(config, invalid): with self.subTest(f'invalid: {config}'): - with self.assertRaises(RuntimeError): + with self.assertRaises(_interpreters.InterpreterError): check(config) elif match(config, questionable): with self.subTest(f'questionable: {config}'): @@ -2427,7 +2427,7 @@ def new_interp(config): with self.subTest('main'): expected = _interpreters.new_config('legacy') expected.gil = 'own' - interpid = _interpreters.get_main() + interpid, *_ = _interpreters.get_main() config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) From 4421169f681f85f8011a85da22616943b6c7a302 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 9 Apr 2024 09:48:13 -0600 Subject: [PATCH 20/28] Fix test_capi. --- Lib/test/test_capi/test_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index a76b866f25f804..35d6a209122a99 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2065,7 +2065,7 @@ def test_configured_settings(self): _testinternalcapi.get_interp_settings() raise NotImplementedError('unreachable') ''') - with self.assertRaises(RuntimeError): + with self.assertRaises(_interpreters.InterpreterError): support.run_in_subinterp_with_config(script, **kwargs) @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") From c75a115c3e6c2f71594fa5191465a075518fb55a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 9 Apr 2024 13:56:47 -0600 Subject: [PATCH 21/28] Add _interpreters.capture_exception(). --- Include/internal/pycore_crossinterp.h | 5 + Modules/_xxsubinterpretersmodule.c | 77 +++++++++++++- Python/crossinterp.c | 141 +++++++++++++++++++++++++- 3 files changed, 217 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 1d797f81d8b21a..64d881dbab7357 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -217,6 +217,11 @@ typedef struct _excinfo { const char *errdisplay; } _PyXI_excinfo; +PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc); +PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info); +PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info); +PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info); + typedef enum error_code { _PyXI_ERR_NO_ERROR = 0, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 240ad46ef4b5f5..7cebe3f406b3a5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1199,9 +1199,78 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) } +static PyObject * +capture_exception(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"exc", NULL}; + PyObject *exc_arg = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "|O:capture_exception", kwlist, + &exc_arg)) + { + return NULL; + } + + PyObject *exc = exc_arg; + if (exc == NULL || exc == Py_None) { + exc = PyErr_GetRaisedException(); + if (exc == NULL) { + Py_RETURN_NONE; + } + } + else if (!PyExceptionInstance_Check(exc)) { + PyErr_Format(PyExc_TypeError, "expected exception, got %R", exc); + return NULL; + } + PyObject *captured = NULL; + + _PyXI_excinfo info = {0}; + if (_PyXI_InitExcInfo(&info, exc) < 0) { + goto finally; + } + captured = _PyXI_ExcInfoAsObject(&info); + if (captured == NULL) { + goto finally; + } + + PyObject *formatted = _PyXI_FormatExcInfo(&info); + if (formatted == NULL) { + Py_CLEAR(captured); + goto finally; + } + int res = PyObject_SetAttrString(captured, "formatted", formatted); + Py_DECREF(formatted); + if (res < 0) { + Py_CLEAR(captured); + goto finally; + } + +finally: + _PyXI_ClearExcInfo(&info); + if (exc != exc_arg) { + if (PyErr_Occurred()) { + PyErr_SetRaisedException(exc); + } + else { + _PyErr_ChainExceptions1(exc); + } + } + return captured; +} + +PyDoc_STRVAR(capture_exception_doc, +"capture_exception(exc=None) -> types.SimpleNamespace\n\ +\n\ +Return a snapshot of an exception. If \"exc\" is None\n\ +then the current exception, if any, is used (but not cleared).\n\ +\n\ +The returned snapshot is the same as what _interpreters.exec() returns."); + + static PyMethodDef module_functions[] = { {"new_config", _PyCFunction_CAST(interp_new_config), METH_VARARGS | METH_KEYWORDS, new_config_doc}, + {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), @@ -1228,14 +1297,18 @@ static PyMethodDef module_functions[] = { {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), METH_VARARGS, set___main___attrs_doc}, - {"is_shareable", _PyCFunction_CAST(object_is_shareable), - METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, {"incref", _PyCFunction_CAST(interp_incref), METH_VARARGS | METH_KEYWORDS, NULL}, {"decref", _PyCFunction_CAST(interp_decref), METH_VARARGS | METH_KEYWORDS, NULL}, + {"is_shareable", _PyCFunction_CAST(object_is_shareable), + METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + + {"capture_exception", _PyCFunction_CAST(capture_exception), + METH_VARARGS | METH_KEYWORDS, capture_exception_doc}, + {NULL, NULL} /* sentinel */ }; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 5a07e45b60679d..64000e1a58b4ef 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -468,7 +468,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int rawfree) /***********************/ static int -_excinfo_init_type(struct _excinfo_type *info, PyObject *exc) +_excinfo_init_type_from_exception(struct _excinfo_type *info, PyObject *exc) { /* Note that this copies directly rather than into an intermediate struct and does not clear on error. If we need that then we @@ -504,7 +504,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) } info->qualname = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); - if (info->name == NULL) { + if (info->qualname == NULL) { return -1; } @@ -515,10 +515,51 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) } info->module = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); + if (info->module == NULL) { + return -1; + } + + return 0; +} + +static int +_excinfo_init_type_from_object(struct _excinfo_type *info, PyObject *exctype) +{ + PyObject *strobj = NULL; + + // __name__ + strobj = PyObject_GetAttrString(exctype, "__name__"); + if (strobj == NULL) { + return -1; + } + info->name = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); if (info->name == NULL) { return -1; } + // __qualname__ + strobj = PyObject_GetAttrString(exctype, "__qualname__"); + if (strobj == NULL) { + return -1; + } + info->qualname = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->qualname == NULL) { + return -1; + } + + // __module__ + strobj = PyObject_GetAttrString(exctype, "__module__"); + if (strobj == NULL) { + return -1; + } + info->module = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->module == NULL) { + return -1; + } + return 0; } @@ -584,7 +625,7 @@ _PyXI_excinfo_Clear(_PyXI_excinfo *info) *info = (_PyXI_excinfo){{NULL}}; } -static PyObject * +PyObject * _PyXI_excinfo_format(_PyXI_excinfo *info) { const char *module, *qualname; @@ -627,7 +668,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) } const char *failure = NULL; - if (_excinfo_init_type(&info->type, exc) < 0) { + if (_excinfo_init_type_from_exception(&info->type, exc) < 0) { failure = "error while initializing exception type snapshot"; goto error; } @@ -672,6 +713,57 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) return failure; } +static const char * +_PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj) +{ + const char *failure = NULL; + + PyObject *exctype = PyObject_GetAttrString(obj, "type"); + if (exctype == NULL) { + failure = "exception snapshot missing 'type' attribute"; + goto error; + } + int res = _excinfo_init_type_from_object(&info->type, exctype); + Py_DECREF(exctype); + if (res < 0) { + failure = "error while initializing exception type snapshot"; + goto error; + } + + // Extract the exception message. + PyObject *msgobj = PyObject_GetAttrString(obj, "msg"); + if (msgobj == NULL) { + failure = "exception snapshot missing 'msg' attribute"; + goto error; + } + info->msg = _copy_string_obj_raw(msgobj, NULL); + Py_DECREF(msgobj); + if (info->msg == NULL) { + failure = "error while copying exception message"; + goto error; + } + + // Pickle a traceback.TracebackException. + PyObject *errdisplay = PyObject_GetAttrString(obj, "errdisplay"); + if (errdisplay == NULL) { + failure = "exception snapshot missing 'errdisplay' attribute"; + goto error; + } + info->errdisplay = _copy_string_obj_raw(errdisplay, NULL); + Py_DECREF(errdisplay); + if (info->errdisplay == NULL) { + failure = "error while copying exception error display"; + goto error; + } + + return NULL; + +error: + assert(failure != NULL); + _PyXI_excinfo_Clear(info); + return failure; +} + static void _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) { @@ -825,6 +917,47 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info) } +int +_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) +{ + assert(!PyErr_Occurred()); + if (exc == NULL || exc == Py_None) { + PyErr_SetString(PyExc_ValueError, "missing exc"); + return -1; + } + const char *failure; + if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) { + failure = _PyXI_excinfo_InitFromException(info, exc); + } + else { + failure = _PyXI_excinfo_InitFromObject(info, exc); + } + if (failure != NULL) { + PyErr_SetString(PyExc_Exception, failure); + return -1; + } + return 0; +} + +PyObject * +_PyXI_FormatExcInfo(_PyXI_excinfo *info) +{ + return _PyXI_excinfo_format(info); +} + +PyObject * +_PyXI_ExcInfoAsObject(_PyXI_excinfo *info) +{ + return _PyXI_excinfo_AsObject(info); +} + +void +_PyXI_ClearExcInfo(_PyXI_excinfo *info) +{ + _PyXI_excinfo_Clear(info); +} + + /***************************/ /* short-term data sharing */ /***************************/ From b00476d579a4d3d26e87c3a15ea5dd03fa7c4dc5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 9 Apr 2024 14:34:21 -0600 Subject: [PATCH 22/28] Raise ExecutionFailed when possible. --- Lib/test/test_interpreters/test_api.py | 53 +++++++++--------- Lib/test/test_interpreters/utils.py | 75 +++++++++++++++----------- 2 files changed, 70 insertions(+), 58 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index b4b1b69cdc1edc..16bb478536aeba 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -169,12 +169,11 @@ def test_created_with_capi(self): for id, *_ in _interpreters.list_all(): last = max(last, id) expected = _testinternalcapi.next_interpreter_id() - err, text = self.run_temp_from_capi(f""" + text = self.run_temp_from_capi(f""" import {interpreters.__name__} as interpreters interp = interpreters.get_current() print(interp.id) """) - assert err is None, err interpid = eval(text) self.assertEqual(interpid, expected) @@ -236,13 +235,12 @@ def test_created_with_capi(self): (interpid5,), ] expected2 = expected[:-2] - err, text = self.run_temp_from_capi(f""" + text = self.run_temp_from_capi(f""" import {interpreters.__name__} as interpreters interp = interpreters.create() print( [(i.id,) for i in interpreters.list_all()]) """) - assert err is None, err res = eval(text) res2 = [(i.id,) for i in interpreters.list_all()] self.assertEqual(res, expected) @@ -393,8 +391,7 @@ def test_created_with_capi(self): interp = interpreters.get_current() print(interp.is_running()) """) - def resolve_results(err, text): - assert err is None, err + def parse_results(text): self.assertNotEqual(text, "") try: return eval(text) @@ -403,13 +400,13 @@ def resolve_results(err, text): with self.subTest('running __main__ (from self)'): with self.interpreter_from_capi() as interpid: - err, text = self.run_from_capi(interpid, script, main=True) - running = resolve_results(err, text) + text = self.run_from_capi(interpid, script, main=True) + running = parse_results(text) self.assertTrue(running) with self.subTest('running, but not __main__ (from self)'): - err, text = self.run_temp_from_capi(script) - running = resolve_results(err, text) + text = self.run_temp_from_capi(script) + running = parse_results(text) self.assertFalse(running) with self.subTest('running __main__ (from other)'): @@ -566,20 +563,17 @@ def test_created_with_capi(self): interp = interpreters.get_current() interp.close() """) - def check_results(err, text): - self.assertIsNot(err, None) - self.assertEqual(err.type.__name__, 'InterpreterError') - self.assertIn('current', err.msg) - self.assertEqual(text, '') with self.subTest('running __main__ (from self)'): with self.interpreter_from_capi() as interpid: - err, text = self.run_from_capi(interpid, script, main=True) - check_results(err, text) + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*current'): + self.run_from_capi(interpid, script, main=True) with self.subTest('running, but not __main__ (from self)'): - err, text = self.run_temp_from_capi(script) - check_results(err, text) + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*current'): + self.run_temp_from_capi(script) with self.subTest('running __main__ (from other)'): with self.interpreter_obj_from_capi() as (interp, interpid): @@ -691,7 +685,9 @@ def test_success(self): script, results = _captured_script('print("it worked!", end="")') with results: interp.exec(script) - out = results.stdout() + results = results.final() + results.raise_if_failed() + out = results.stdout self.assertEqual(out, 'it worked!') @@ -758,7 +754,9 @@ def f(): t = threading.Thread(target=f) t.start() t.join() - out = results.stdout() + results = results.final() + results.raise_if_failed() + out = results.stdout self.assertEqual(out, 'it worked!') @@ -1278,8 +1276,7 @@ def parse_stdout(text): for id, *_ in _interpreters.list_all(): last = max(last, id) expected = last + 1 - err, text = self.run_temp_from_capi(script) - assert err is None, err + text = self.run_temp_from_capi(script) interpid, = parse_stdout(text) self.assertEqual(interpid, expected) @@ -1319,13 +1316,12 @@ def test_list_all(self): expected3 = expected + [ (interpid5,), ] - err, text = self.run_temp_from_capi(f""" + text = self.run_temp_from_capi(f""" import {_interpreters.__name__} as _interpreters _interpreters.create() print( _interpreters.list_all()) """) - assert err is None, err res2 = eval(text) res3 = _interpreters.list_all() self.assertEqual(res2, expected2) @@ -1483,8 +1479,9 @@ def test_exec(self): script, results = _captured_script('print("it worked!", end="")') with results: exc = _interpreters.exec(interpid, script) - out = results.stdout() - self.assertIs(exc, None) + results = results.final() + results.raise_if_failed() + out = results.stdout self.assertEqual(out, 'it worked!') with self.subTest('uncaught exception'): @@ -1497,7 +1494,7 @@ def test_exec(self): exc = _interpreters.exec(interpid, script) out = results.stdout() self.assertEqual(out, '') - self.assertEqual(exc, types.SimpleNamespace( + self.assert_ns_equal(exc, types.SimpleNamespace( type=types.SimpleNamespace( __name__='Exception', __qualname__='Exception', diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index d4ee2e1562f441..039defc9282a54 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -56,10 +56,28 @@ def _close_file(file): # It was closed already. +def pack_exception(exc=None): + captured = _interpreters.capture_exception(exc) + data = dict(captured.__dict__) + data['type'] = dict(captured.type.__dict__) + return json.dumps(data) + + +def unpack_exception(packed): + try: + data = json.loads(packed) + except json.decoder.JSONDecodeError: + warnings.warn('incomplete exception data', RuntimeWarning) + print(packed if isinstance(packed, str) else packed.decode('utf-8')) + return None + exc = types.SimpleNamespace(**data) + exc.type = types.SimpleNamespace(**exc.type) + return exc; + + class CapturingResults: STDIO = dedent("""\ - import contextlib, io with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}: _captured_std{stream} = io.StringIO() with contextlib.redirect_std{stream}(_captured_std{stream}): @@ -74,7 +92,6 @@ class CapturingResults: _spipe_{stream}.write(text.encode('utf-8')) """)[:-1] EXC = dedent("""\ - import json, traceback with open({w_pipe}, 'wb', buffering=0) as _spipe_exc: try: ######################### @@ -85,23 +102,16 @@ class CapturingResults: # end wrapped script ######################### except Exception as exc: - # This matches what _interpreters.exec() returns. - text = json.dumps(dict( - type=dict( - __name__=type(exc).__name__, - __qualname__=type(exc).__qualname__, - __module__=type(exc).__module__, - ), - msg=str(exc), - formatted=traceback.format_exception_only(exc), - errdisplay=traceback.format_exception(exc), - )) + text = _interp_utils.pack_exception(exc) _spipe_exc.write(text.encode('utf-8')) """)[:-1] @classmethod def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): script = dedent(script).strip(os.linesep) + imports = [ + f'import {__name__} as _interp_utils', + ] wrapped = script # Handle exc. @@ -118,6 +128,9 @@ def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): # Handle stdout. if stdout: + imports.extend([ + 'import contextlib, io', + ]) stdout = os.pipe() r_out, w_out = stdout indented = wrapped.replace('\n', '\n ') @@ -133,6 +146,10 @@ def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): if stderr == 'stdout': stderr = None elif stderr: + if not stdout: + imports.extend([ + 'import contextlib, io', + ]) stderr = os.pipe() r_err, w_err = stderr indented = wrapped.replace('\n', '\n ') @@ -146,6 +163,10 @@ def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): if wrapped == script: raise NotImplementedError + else: + for line in imports: + wrapped = f'{line}{os.linesep}{wrapped}' + results = cls(stdout, stderr, exc) return wrapped, results @@ -246,14 +267,7 @@ def _unpack_exc(self): return self._exc if not self._buf_exc: return None - try: - data = json.loads(self._buf_exc) - except json.decoder.JSONDecodeError: - warnings.warn('incomplete exception data', RuntimeWarning) - print(self._buf_exc.decode('utf-8')) - return None - self._exc = exc = types.SimpleNamespace(**data) - exc.type = types.SimpleNamespace(**exc.type) + self._exc = unpack_exception(self._buf_exc) return self._exc def stdout(self): @@ -313,6 +327,10 @@ def __getattr__(self, name): raise AttributeError(name) return getattr(self._final, name) + def raise_if_failed(self): + if self.exc is not None: + raise interpreters.ExecutionFailed(self.exc) + def _captured_script(script, *, stdout=True, stderr=False, exc=False): return CapturingResults.wrap_script( @@ -487,11 +505,7 @@ def _run_string(self, interp, script): def run_and_capture(self, interp, script): text, err = self._run_string(interp, script) if err is not None: - print() - if not err.errdisplay.startswith('Traceback '): - print('Traceback (most recent call last):') - print(err.errdisplay, file=sys.stderr) - raise Exception(f'subinterpreter failed: {err.formatted}') + raise interpreters.ExecutionFailed(err) else: return text @@ -526,7 +540,8 @@ def run_from_capi(self, interpid, script, *, main=False): with self.capturing(script) as (wrapped, results): rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main) assert rc == 0, rc - return results.exc, results.stdout + results.raise_if_failed() + return results.stdout @contextlib.contextmanager def _running(self, run_interp, exec_interp): @@ -604,8 +619,7 @@ def exec_interp(script): @contextlib.contextmanager def running_from_capi(self, interpid, *, main=False): def run_interp(script): - err, text = self.run_from_capi(interpid, script, main=main) - assert err is None, err + text = self.run_from_capi(interpid, script, main=main) assert text == '', repr(text) def exec_interp(script): rc = _testinternalcapi.exec_interpreter(interpid, script) @@ -620,4 +634,5 @@ def run_temp_from_capi(self, script, config='legacy'): with self.capturing(script) as (wrapped, results): rc = run_in_interpreter(wrapped, config) assert rc == 0, rc - return results.exc, results.stdout + results.raise_if_failed() + return results.stdout From 6a82a33f7a083151ecab03ad18d2a50532ec4d42 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 9 Apr 2024 14:44:08 -0600 Subject: [PATCH 23/28] Handle OSError in the _running() script. --- Lib/test/test_interpreters/utils.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 039defc9282a54..4acc850bea9f63 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -558,11 +558,18 @@ def close(): # Start running (and wait). script = dedent(f""" import os - # handshake - token = os.read({r_in}, 1) - os.write({w_out}, token) - # Wait for the "done" message. - os.read({r_in}, 1) + try: + # handshake + token = os.read({r_in}, 1) + os.write({w_out}, token) + # Wait for the "done" message. + os.read({r_in}, 1) + except BrokenPipeError: + pass + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. """) failed = None def run(): From 11dae3d198a56f3a03615bc60d5dffcda3e45550 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 10 Apr 2024 14:38:16 -0600 Subject: [PATCH 24/28] Add PyInterpreterState._whence. --- Include/internal/pycore_interp.h | 14 +++++++++ Include/internal/pycore_runtime_init.h | 1 + Modules/_xxsubinterpretersmodule.c | 41 ++++++++++++++++++++++++++ Python/crossinterp.c | 2 ++ Python/pylifecycle.c | 11 +++++-- Python/pystate.c | 41 ++++++++++++++++++++++++-- 6 files changed, 105 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index b8d0fdcce11ba8..f7801614358a5d 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -97,6 +97,15 @@ struct _is { int requires_idref; PyThread_type_lock id_mutex; +#define _PyInterpreterState_WHENCE_NOTSET -1 +#define _PyInterpreterState_WHENCE_UNKNOWN 0 +#define _PyInterpreterState_WHENCE_RUNTIME 1 +#define _PyInterpreterState_WHENCE_LEGACY_CAPI 2 +#define _PyInterpreterState_WHENCE_CAPI 3 +#define _PyInterpreterState_WHENCE_XI 4 +#define _PyInterpreterState_WHENCE_MAX 4 + long _whence; + /* Has been initialized to a safe state. In order to be effective, this must be set to 0 during or right @@ -304,6 +313,11 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); +PyAPI_FUNC(long) _PyInterpreterState_GetWhence(PyInterpreterState *interp); +extern void _PyInterpreterState_SetWhence( + PyInterpreterState *interp, + long whence); + extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); // Get a copy of the current interpreter configuration. diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 88d888943d28b1..33c7a9dadfd2a1 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -162,6 +162,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyInterpreterState_INIT(INTERP) \ { \ .id_refcount = -1, \ + ._whence = _PyInterpreterState_WHENCE_NOTSET, \ .imports = IMPORTS_INIT, \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 7cebe3f406b3a5..37ac5a3f28aba9 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1151,6 +1151,32 @@ PyDoc_STRVAR(get_config_doc, Return a representation of the config used to initialize the interpreter."); +static PyObject * +interp_whence(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *id; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:whence", kwlist, &id)) + { + return NULL; + } + + PyInterpreterState *interp = look_up_interp(id); + if (interp == NULL) { + return NULL; + } + + long whence = _PyInterpreterState_GetWhence(interp); + return PyLong_FromLong(whence); +} + +PyDoc_STRVAR(whence_doc, +"whence(id) -> int\n\ +\n\ +Return an identifier for where the interpreter was created."); + + static PyObject * interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1286,6 +1312,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, is_running_doc}, {"get_config", _PyCFunction_CAST(interp_get_config), METH_VARARGS | METH_KEYWORDS, get_config_doc}, + {"whence", _PyCFunction_CAST(interp_whence), + METH_VARARGS | METH_KEYWORDS, whence_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, {"call", _PyCFunction_CAST(interp_call), @@ -1325,6 +1353,19 @@ module_exec(PyObject *mod) PyInterpreterState *interp = PyInterpreterState_Get(); module_state *state = get_module_state(mod); +#define ADD_WHENCE(NAME) \ + if (PyModule_AddIntConstant(mod, "WHENCE_" #NAME, \ + _PyInterpreterState_WHENCE_##NAME) < 0) \ + { \ + goto error; \ + } + ADD_WHENCE(UNKNOWN) + ADD_WHENCE(RUNTIME) + ADD_WHENCE(LEGACY_CAPI) + ADD_WHENCE(CAPI) + ADD_WHENCE(XI) +#undef ADD_WHENCE + // exceptions if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { goto error; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 64000e1a58b4ef..ae6955b1366985 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1845,6 +1845,8 @@ _PyXI_NewInterpreter(PyInterpreterConfig *config, assert(tstate != NULL); PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_XI); + if (p_tstate != NULL) { // We leave the new thread state as the current one. *p_tstate = tstate; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 1d315b80d88ce0..b414f35a5f5c0d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -631,6 +631,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, } assert(interp != NULL); assert(_Py_IsMainInterpreter(interp)); + _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME); status = _PyConfig_Copy(&interp->config, src_config); if (_PyStatus_EXCEPTION(status)) { @@ -2120,7 +2121,8 @@ Py_Finalize(void) */ static PyStatus -new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) +new_interpreter(PyThreadState **tstate_p, + const PyInterpreterConfig *config, long whence) { PyStatus status; @@ -2143,6 +2145,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) *tstate_p = NULL; return _PyStatus_OK(); } + _PyInterpreterState_SetWhence(interp, whence); // XXX Might new_interpreter() have been called without the GIL held? PyThreadState *save_tstate = _PyThreadState_GET(); @@ -2231,15 +2234,17 @@ PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) { - return new_interpreter(tstate_p, config); + long whence = _PyInterpreterState_WHENCE_CAPI; + return new_interpreter(tstate_p, config, whence); } PyThreadState * Py_NewInterpreter(void) { PyThreadState *tstate = NULL; + long whence = _PyInterpreterState_WHENCE_LEGACY_CAPI; const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - PyStatus status = new_interpreter(&tstate, &config); + PyStatus status = new_interpreter(&tstate, &config, whence); if (_PyStatus_EXCEPTION(status)) { Py_ExitStatusException(status); } diff --git a/Python/pystate.c b/Python/pystate.c index fecf6cb6ca506e..6fc5930b51ae6a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -574,6 +574,8 @@ free_interpreter(PyInterpreterState *interp) } } +static inline int check_interpreter_whence(long); + /* Get the interpreter state to a minimal consistent state. Further init happens in pylifecycle.c before it can be used. All fields not initialized here are expected to be zeroed out, @@ -596,12 +598,17 @@ free_interpreter(PyInterpreterState *interp) static PyStatus init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, - PyInterpreterState *next) + PyInterpreterState *next, + long whence) { if (interp->_initialized) { return _PyStatus_ERR("interpreter already initialized"); } + assert(interp->_whence == _PyInterpreterState_WHENCE_NOTSET); + assert(check_interpreter_whence(whence) == 0); + interp->_whence = whence; + assert(runtime != NULL); interp->runtime = runtime; @@ -709,8 +716,9 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) } interpreters->head = interp; + long whence = _PyInterpreterState_WHENCE_UNKNOWN; status = init_interpreter(interp, runtime, - id, old_head); + id, old_head, whence); if (_PyStatus_EXCEPTION(status)) { goto error; } @@ -1094,6 +1102,34 @@ _PyInterpreterState_ReinitRunningMain(PyThreadState *tstate) // accessors //---------- +static inline int +check_interpreter_whence(long whence) +{ + if(whence < 0) { + return -1; + } + if (whence > _PyInterpreterState_WHENCE_MAX) { + return -1; + } + return 0; +} + +long +_PyInterpreterState_GetWhence(PyInterpreterState *interp) +{ + assert(check_interpreter_whence(interp->_whence) == 0); + return interp->_whence; +} + +void +_PyInterpreterState_SetWhence(PyInterpreterState *interp, long whence) +{ + assert(interp->_whence != _PyInterpreterState_WHENCE_NOTSET); + assert(check_interpreter_whence(whence) == 0); + interp->_whence = whence; +} + + PyObject * PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) { @@ -1105,6 +1141,7 @@ PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) return PyMapping_GetItemString(modules, "__main__"); } + PyObject * PyInterpreterState_GetDict(PyInterpreterState *interp) { From 7c9a2b95cc10f660b524fb246d294ce17c8de8a4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 10 Apr 2024 14:39:06 -0600 Subject: [PATCH 25/28] Fix up _testinternalcapi. --- Modules/_testinternalcapi.c | 171 ++++++++++++++++++++++++++++++------ 1 file changed, 143 insertions(+), 28 deletions(-) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 8df66713233560..060c01db092ee7 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1357,28 +1357,81 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } +static int +_init_interp_config_from_object(PyInterpreterConfig *config, PyObject *obj) +{ + if (obj == NULL) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + return 0; + } + + PyObject *dict = PyObject_GetAttrString(obj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", obj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + return 0; +} + static PyInterpreterState * -_new_interpreter(PyObject *configobj, - PyThreadState **p_tstate, PyThreadState **p_save_tstate) +_new_interpreter(PyInterpreterConfig *config, long whence) { - PyInterpreterConfig config; - if (configobj == NULL) { - config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + if (whence == _PyInterpreterState_WHENCE_XI) { + return _PyXI_NewInterpreter(config, NULL, NULL); } - else { - PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); - if (dict == NULL) { - PyErr_Format(PyExc_TypeError, "bad config %R", configobj); - return NULL; + PyObject *exc = NULL; + PyInterpreterState *interp = NULL; + if (whence == _PyInterpreterState_WHENCE_UNKNOWN) { + assert(config == NULL); + interp = PyInterpreterState_New(); + } + else if (whence == _PyInterpreterState_WHENCE_CAPI + || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) + { + PyThreadState *tstate = NULL; + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + if (whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) { + assert(config == NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); } - int res = _PyInterpreterConfig_InitFromDict(&config, dict); - Py_DECREF(dict); - if (res < 0) { - return NULL; + else { + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + assert(tstate == NULL); + _PyErr_SetFromPyStatus(status); + exc = PyErr_GetRaisedException(); + } + } + if (tstate != NULL) { + interp = PyThreadState_GetInterpreter(tstate); + // Throw away the initial tstate. + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); } } + else { + PyErr_Format(PyExc_ValueError, + "unsupported whence %ld", whence); + return NULL; + } - return _PyXI_NewInterpreter(&config, p_tstate, p_save_tstate); + if (interp == NULL) { + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + if (exc != NULL) { + _PyErr_ChainExceptions1(exc); + } + } + return interp; } // This exists mostly for testing the _interpreters module, as an @@ -1386,20 +1439,41 @@ _new_interpreter(PyObject *configobj, static PyObject * create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) { - static char *kwlist[] = {"config", NULL}; + static char *kwlist[] = {"config", "whence", NULL}; PyObject *configobj = NULL; + long whence = _PyInterpreterState_WHENCE_XI; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|O:create_interpreter", kwlist, - &configobj)) + "|O$p:create_interpreter", kwlist, + &configobj, &whence)) { return NULL; } - PyInterpreterState *interp = _new_interpreter(configobj, NULL, NULL); + // Resolve the config. + PyInterpreterConfig *config = NULL; + PyInterpreterConfig _config; + if (whence == _PyInterpreterState_WHENCE_UNKNOWN + || whence == _PyInterpreterState_WHENCE_UNKNOWN) + { + if (configobj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected config"); + return NULL; + } + } + else { + config = &_config; + if (_init_interp_config_from_object(config, configobj) < 0) { + return NULL; + } + } + + // Create the interpreter. + PyInterpreterState *interp = _new_interpreter(config, whence); if (interp == NULL) { return NULL; } + // Return the ID. PyObject *idobj = _PyInterpreterState_GetIDObject(interp); if (idobj == NULL) { _PyXI_EndInterpreter(interp, NULL, NULL); @@ -1497,26 +1571,67 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) { const char *code; PyObject *configobj; - static char *kwlist[] = {"code", "config", NULL}; + int xi = 0; + static char *kwlist[] = {"code", "config", "xi", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "sO:run_in_subinterp_with_config", kwlist, - &code, &configobj)) + "sO|$p:run_in_subinterp_with_config", kwlist, + &code, &configobj, &xi)) { return NULL; } - PyThreadState *save_tstate; - PyThreadState *substate; - PyInterpreterState *interp = _new_interpreter(configobj, &substate, &save_tstate); - if (interp == NULL) { + PyInterpreterConfig config; + if (_init_interp_config_from_object(&config, configobj) < 0) { return NULL; } /* only initialise 'cflags.cf_flags' to test backwards compatibility */ PyCompilerFlags cflags = {0}; - int r = PyRun_SimpleStringFlags(code, &cflags); - _PyXI_EndInterpreter(interp, substate, &save_tstate); + int r; + if (xi) { + PyThreadState *save_tstate; + PyThreadState *tstate; + + /* Create an interpreter, staying switched to it. */ + PyInterpreterState *interp = \ + _PyXI_NewInterpreter(&config, &tstate, &save_tstate); + if (interp == NULL) { + return NULL; + } + + /* Exec the code in the new interpreter. */ + r = PyRun_SimpleStringFlags(code, &cflags); + + /* clean up post-exec. */ + _PyXI_EndInterpreter(interp, tstate, &save_tstate); + } + else { + PyThreadState *substate; + PyThreadState *mainstate = PyThreadState_Swap(NULL); + + /* Create an interpreter, staying switched to it. */ + PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyThreadState_Swap(mainstate); + _PyErr_SetFromPyStatus(status); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } + + /* Exec the code in the new interpreter. */ + r = PyRun_SimpleStringFlags(code, &cflags); + + /* clean up post-exec. */ + Py_EndInterpreter(substate); + PyThreadState_Swap(mainstate); + } return PyLong_FromLong(r); } From e68654cfcc35c5810bf4150feef67e578d0e378c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 10 Apr 2024 15:18:37 -0600 Subject: [PATCH 26/28] Add PyInterpreterState.initialized. --- Include/internal/pycore_interp.h | 2 ++ Python/pylifecycle.c | 1 + 2 files changed, 3 insertions(+) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index f7801614358a5d..42ac90fab2d4a2 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -111,6 +111,8 @@ struct _is { In order to be effective, this must be set to 0 during or right after allocation. */ int _initialized; + /* Has been fully initialized via pylifecycle.c. */ + int _ready; int finalizing; uintptr_t last_restart_version; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b414f35a5f5c0d..76806dbf8041bd 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2146,6 +2146,7 @@ new_interpreter(PyThreadState **tstate_p, return _PyStatus_OK(); } _PyInterpreterState_SetWhence(interp, whence); + interp->_ready = 1; // XXX Might new_interpreter() have been called without the GIL held? PyThreadState *save_tstate = _PyThreadState_GET(); From 175080c83444dc3515602b2a9bddce788a213442 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 10 Apr 2024 16:03:04 -0600 Subject: [PATCH 27/28] Add tests for whence. --- Lib/test/test_interpreters/test_api.py | 52 +++++++++++++++++++++++-- Lib/test/test_interpreters/utils.py | 53 +++++++++++++++++++------- Modules/_testinternalcapi.c | 7 +++- Python/crossinterp.c | 15 +++++++- 4 files changed, 106 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 16bb478536aeba..abf66a7cde796c 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -16,7 +16,7 @@ ) from .utils import ( _captured_script, _run_output, _running, TestBase, - requires__testinternalcapi, _testinternalcapi, + requires_test_modules, _testinternalcapi, ) @@ -163,7 +163,7 @@ def test_idempotent(self): id2 = id(interp) self.assertNotEqual(id1, id2) - @requires__testinternalcapi + @requires_test_modules def test_created_with_capi(self): last = 0 for id, *_ in _interpreters.list_all(): @@ -668,7 +668,7 @@ def test_running(self): interp.prepare_main({'spam': False}) interp.exec('assert spam is True') - @requires__testinternalcapi + @requires_test_modules def test_created_with_capi(self): with self.interpreter_from_capi() as interpid: interp = interpreters.Interpreter(interpid) @@ -1378,7 +1378,7 @@ def test_create(self): with self.assertRaises(ValueError): _interpreters.create(orig) - @requires__testinternalcapi + @requires_test_modules def test_destroy(self): with self.subTest('from _interpreters'): interpid = _interpreters.create() @@ -1439,6 +1439,50 @@ def test_get_config(self): config = _interpreters.get_config(interpid) self.assert_ns_equal(config, orig) + @requires_test_modules + def test_whence(self): + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + with self.subTest('stdlib'): + interpid = _interpreters.create() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_XI) + + for orig, name in { + # XXX Also check WHENCE_UNKNOWN. + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + }.items(): + with self.subTest(f'from C-API ({orig}: {name})'): + with self.interpreter_from_capi(whence=orig) as interpid: + whence = _interpreters.whence(interpid) + self.assertEqual(whence, orig) + + with self.subTest('from C-API, running'): + text = self.run_temp_from_capi(dedent(f""" + import {_interpreters.__name__} as _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=True) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + with self.subTest('from legacy C-API, running'): + ... + text = self.run_temp_from_capi(dedent(f""" + import {_interpreters.__name__} as _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=False) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI) + def test_is_running(self): with self.subTest('main'): interpid, *_ = _interpreters.get_main() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 4acc850bea9f63..d92179474959ef 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -26,12 +26,12 @@ try: import _testinternalcapi + import _testcapi except ImportError: _testinternalcapi = None -else: - run_in_interpreter = _testinternalcapi.run_in_subinterp_with_config + _testcapi = None -def requires__testinternalcapi(func): +def requires_test_modules(func): return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) @@ -509,12 +509,31 @@ def run_and_capture(self, interp, script): else: return text - @requires__testinternalcapi + @requires_test_modules @contextlib.contextmanager - def interpreter_from_capi(self, config='legacy'): - if isinstance(config, str): + def interpreter_from_capi(self, config=None, whence=None): + if config is False: + if whence is None: + whence = _interpreters.WHENCE_LEGACY_CAPI + else: + assert whence in (_interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN), repr(whence) + config = None + elif config is True: + config = _interpreters.new_config('default') + elif config is None: + if whence not in ( + _interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN, + ): + config = _interpreters.new_config('legacy') + elif isinstance(config, str): config = _interpreters.new_config(config) - interpid = _testinternalcapi.create_interpreter() + + if whence is None: + whence = _interpreters.WHENCE_XI + + interpid = _testinternalcapi.create_interpreter(config, whence=whence) try: yield interpid finally: @@ -535,7 +554,7 @@ def capturing(self, script): with capturing: yield wrapped, capturing.final(force=True) - @requires__testinternalcapi + @requires_test_modules def run_from_capi(self, interpid, script, *, main=False): with self.capturing(script) as (wrapped, results): rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main) @@ -622,7 +641,7 @@ def exec_interp(script): with self._running(run_interp, exec_interp): yield - @requires__testinternalcapi + @requires_test_modules @contextlib.contextmanager def running_from_capi(self, interpid, *, main=False): def run_interp(script): @@ -634,12 +653,20 @@ def exec_interp(script): with self._running(run_interp, exec_interp): yield - @requires__testinternalcapi + @requires_test_modules def run_temp_from_capi(self, script, config='legacy'): - if isinstance(config, str): - config = _interpreters.new_config(config) + if config is False: + # Force using Py_NewInterpreter(). + run_in_interp = (lambda s, c: _testcapi.run_in_subinterp(s)) + config = None + else: + run_in_interp = _testinternalcapi.run_in_subinterp_with_config + if config is True: + config = 'default' + if isinstance(config, str): + config = _interpreters.new_config(config) with self.capturing(script) as (wrapped, results): - rc = run_in_interpreter(wrapped, config) + rc = run_in_interp(wrapped, config) assert rc == 0, rc results.raise_if_failed() return results.stdout diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 060c01db092ee7..a2596091fc9256 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1443,17 +1443,20 @@ create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *configobj = NULL; long whence = _PyInterpreterState_WHENCE_XI; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|O$p:create_interpreter", kwlist, + "|O$l:create_interpreter", kwlist, &configobj, &whence)) { return NULL; } + if (configobj == Py_None) { + configobj = NULL; + } // Resolve the config. PyInterpreterConfig *config = NULL; PyInterpreterConfig _config; if (whence == _PyInterpreterState_WHENCE_UNKNOWN - || whence == _PyInterpreterState_WHENCE_UNKNOWN) + || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) { if (configobj != NULL) { PyErr_SetString(PyExc_ValueError, "got unexpected config"); diff --git a/Python/crossinterp.c b/Python/crossinterp.c index ae6955b1366985..fb0dae0bbb8f75 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1868,8 +1868,8 @@ void _PyXI_EndInterpreter(PyInterpreterState *interp, PyThreadState *tstate, PyThreadState **p_save_tstate) { - PyThreadState *cur_tstate = PyThreadState_GET(); PyThreadState *save_tstate = NULL; + PyThreadState *cur_tstate = PyThreadState_GET(); if (tstate == NULL) { if (PyThreadState_GetInterpreter(cur_tstate) == interp) { tstate = cur_tstate; @@ -1889,7 +1889,18 @@ _PyXI_EndInterpreter(PyInterpreterState *interp, } } - Py_EndInterpreter(tstate); + long whence = _PyInterpreterState_GetWhence(interp); + assert(whence != _PyInterpreterState_WHENCE_RUNTIME); + if (whence == _PyInterpreterState_WHENCE_UNKNOWN) { + assert(!interp->_ready); + PyThreadState *tstate = PyThreadState_New(interp); + save_tstate = PyThreadState_Swap(tstate); + _PyInterpreterState_Clear(tstate); + PyInterpreterState_Delete(interp); + } + else { + Py_EndInterpreter(tstate); + } if (p_save_tstate != NULL) { save_tstate = *p_save_tstate; From 9db5a2b50a7f06bd4d1e949209aca21eb8140b0b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 10 Apr 2024 17:06:15 -0600 Subject: [PATCH 28/28] Set interp-_ready on the main interpreter. --- Python/pylifecycle.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 76806dbf8041bd..4e83b1671a5029 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -477,6 +477,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, if (interp == NULL) { return _PyStatus_ERR("can't make main interpreter"); } + assert(interp->_ready); status = _PyConfig_Write(config, runtime); if (_PyStatus_EXCEPTION(status)) { @@ -632,6 +633,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, assert(interp != NULL); assert(_Py_IsMainInterpreter(interp)); _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME); + interp->_ready = 1; status = _PyConfig_Copy(&interp->config, src_config); if (_PyStatus_EXCEPTION(status)) { pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy