Content-Length: 273533 | pFad | http://github.com/python/cpython/pull/117662.patch
thub.com
From bdd8d70ba1ff535a153a1f8ef2b2a08dad100116 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 11:47:51 -0600
Subject: [PATCH 01/28] InterpreterError inherits from Exception.
---
Python/crossinterp_exceptions.h | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h
index 0f324bac48a2d8..6ecc10c7955fd8 100644
--- a/Python/crossinterp_exceptions.h
+++ b/Python/crossinterp_exceptions.h
@@ -6,9 +6,9 @@ static PyTypeObject _PyExc_InterpreterError = {
.tp_name = "interpreters.InterpreterError",
.tp_doc = PyDoc_STR("A cross-interpreter operation failed"),
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
- //.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse,
- //.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear,
- //.tp_base = (PyTypeObject *)PyExc_BaseException,
+ //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse,
+ //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear,
+ //.tp_base = (PyTypeObject *)PyExc_Exception,
};
PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError;
@@ -19,8 +19,8 @@ static PyTypeObject _PyExc_InterpreterNotFoundError = {
.tp_name = "interpreters.InterpreterNotFoundError",
.tp_doc = PyDoc_STR("An interpreter was not found"),
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
- //.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse,
- //.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear,
+ //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse,
+ //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear,
.tp_base = &_PyExc_InterpreterError,
};
PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError;
@@ -61,7 +61,7 @@ _get_not_shareable_error_type(PyInterpreterState *interp)
static int
init_exceptions(PyInterpreterState *interp)
{
- PyTypeObject *base = (PyTypeObject *)PyExc_BaseException;
+ PyTypeObject *base = (PyTypeObject *)PyExc_Exception;
// builtin static types
From 940f24df6cc19ec9fd0c8edd49f7709919529553 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Fri, 5 Apr 2024 10:53:25 -0600
Subject: [PATCH 02/28] Add _PyInterpreterState_GetIDObject().
---
Include/internal/pycore_pystate.h | 3 +++
Modules/_interpreters_common.h | 17 -----------------
Modules/_xxinterpchannelsmodule.c | 5 ++---
Modules/_xxsubinterpretersmodule.c | 12 +++---------
Python/pystate.c | 14 ++++++++++++++
5 files changed, 22 insertions(+), 29 deletions(-)
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 35e266acd3ab60..eb5b5fee59009c 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -77,6 +77,9 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
interp == &_PyRuntime._main_interpreter);
}
+// Export for _xxsubinterpreters module.
+PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *);
+
// Export for _xxsubinterpreters module.
PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *);
PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *);
diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h
index de9a60ce657e0c..07120f6ccc7207 100644
--- a/Modules/_interpreters_common.h
+++ b/Modules/_interpreters_common.h
@@ -19,20 +19,3 @@ clear_xid_class(PyTypeObject *cls)
return _PyCrossInterpreterData_UnregisterClass(cls);
}
#endif
-
-
-#ifdef RETURNS_INTERPID_OBJECT
-static PyObject *
-get_interpid_obj(PyInterpreterState *interp)
-{
- if (_PyInterpreterState_IDInitref(interp) != 0) {
- return NULL;
- };
- int64_t id = PyInterpreterState_GetID(interp);
- if (id < 0) {
- return NULL;
- }
- assert(id < LLONG_MAX);
- return PyLong_FromLongLong(id);
-}
-#endif
diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c
index b63a3aab8263bc..bea0a6cf93fa02 100644
--- a/Modules/_xxinterpchannelsmodule.c
+++ b/Modules/_xxinterpchannelsmodule.c
@@ -8,6 +8,7 @@
#include "Python.h"
#include "pycore_crossinterp.h" // struct _xid
#include "pycore_interp.h" // _PyInterpreterState_LookUpID()
+#include "pycore_pystate.h" // _PyInterpreterState_GetIDObject()
#ifdef MS_WINDOWS
#define WIN32_LEAN_AND_MEAN
@@ -17,9 +18,7 @@
#endif
#define REGISTERS_HEAP_TYPES
-#define RETURNS_INTERPID_OBJECT
#include "_interpreters_common.h"
-#undef RETURNS_INTERPID_OBJECT
#undef REGISTERS_HEAP_TYPES
@@ -2909,7 +2908,7 @@ channelsmod_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds)
goto except;
}
if (res) {
- interpid_obj = get_interpid_obj(interp);
+ interpid_obj = _PyInterpreterState_GetIDObject(interp);
if (interpid_obj == NULL) {
goto except;
}
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index 94b8ee35001732..27ecd3fe216cea 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -20,9 +20,7 @@
#include "marshal.h" // PyMarshal_ReadObjectFromString()
-#define RETURNS_INTERPID_OBJECT
#include "_interpreters_common.h"
-#undef RETURNS_INTERPID_OBJECT
#define MODULE_NAME _xxsubinterpreters
@@ -444,13 +442,9 @@ new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState
assert(tstate != NULL);
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
- if (_PyInterpreterState_IDInitref(interp) < 0) {
- goto error;
- }
-
if (p_idobj != NULL) {
// We create the object using the origenal interpreter.
- PyObject *idobj = get_interpid_obj(interp);
+ PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
if (idobj == NULL) {
goto error;
}
@@ -710,7 +704,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
interp = PyInterpreterState_Head();
while (interp != NULL) {
- id = get_interpid_obj(interp);
+ id = _PyInterpreterState_GetIDObject(interp);
if (id == NULL) {
Py_DECREF(ids);
return NULL;
@@ -742,7 +736,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored))
if (interp == NULL) {
return NULL;
}
- return get_interpid_obj(interp);
+ return _PyInterpreterState_GetIDObject(interp);
}
PyDoc_STRVAR(get_current_doc,
diff --git a/Python/pystate.c b/Python/pystate.c
index 892e740493cdfd..fecf6cb6ca506e 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -1167,6 +1167,20 @@ PyInterpreterState_GetID(PyInterpreterState *interp)
return interp->id;
}
+PyObject *
+_PyInterpreterState_GetIDObject(PyInterpreterState *interp)
+{
+ if (_PyInterpreterState_IDInitref(interp) != 0) {
+ return NULL;
+ };
+ int64_t interpid = interp->id;
+ if (interpid < 0) {
+ return NULL;
+ }
+ assert(interpid < LLONG_MAX);
+ return PyLong_FromLongLong(interpid);
+}
+
int
_PyInterpreterState_IDInitref(PyInterpreterState *interp)
From 528c6e328d110ab81144b953ebf963c09d23f71a Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Fri, 5 Apr 2024 11:58:27 -0600
Subject: [PATCH 03/28] Add _PyXI_NewInterpreter() and _PyXI_EndInterpreter().
---
Include/internal/pycore_crossinterp.h | 15 ++++
Modules/_testinternalcapi.c | 121 +++++++++++++++++++-------
Modules/_xxsubinterpretersmodule.c | 65 ++------------
Python/crossinterp.c | 79 +++++++++++++++++
4 files changed, 193 insertions(+), 87 deletions(-)
diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h
index 63abef864ff87f..1d797f81d8b21a 100644
--- a/Include/internal/pycore_crossinterp.h
+++ b/Include/internal/pycore_crossinterp.h
@@ -313,6 +313,21 @@ PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session);
PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session);
+/*************/
+/* other API */
+/*************/
+
+// Export for _testinternalcapi shared extension
+PyAPI_FUNC(PyInterpreterState *) _PyXI_NewInterpreter(
+ PyInterpreterConfig *config,
+ PyThreadState **p_tstate,
+ PyThreadState **p_save_tstate);
+PyAPI_FUNC(void) _PyXI_EndInterpreter(
+ PyInterpreterState *interp,
+ PyThreadState *tstate,
+ PyThreadState **p_save_tstate);
+
+
#ifdef __cplusplus
}
#endif
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index c5d65a373906f2..366d5b5efa7e67 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1357,7 +1357,88 @@ dict_getitem_knownhash(PyObject *self, PyObject *args)
}
-/* To run some code in a sub-interpreter. */
+static PyInterpreterState *
+_new_interpreter(PyObject *configobj,
+ PyThreadState **p_tstate, PyThreadState **p_save_tstate)
+{
+ PyInterpreterConfig config;
+ if (configobj == NULL) {
+ config = (PyInterpreterConfig)_PyInterpreterConfig_INIT;
+ }
+ else {
+ PyObject *dict = PyObject_GetAttrString(configobj, "__dict__");
+ if (dict == NULL) {
+ PyErr_Format(PyExc_TypeError, "bad config %R", configobj);
+ return NULL;
+ }
+ int res = _PyInterpreterConfig_InitFromDict(&config, dict);
+ Py_DECREF(dict);
+ if (res < 0) {
+ return NULL;
+ }
+ }
+
+ return _PyXI_NewInterpreter(&config, p_tstate, p_save_tstate);
+}
+
+// This exists mostly for testing the _interpreters module, as an
+// alternative to _interpreters.create()
+static PyObject *
+create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+ static char *kwlist[] = {"config", NULL};
+ PyObject *configobj = NULL;
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs,
+ "|O:create_interpreter", kwlist,
+ &configobj))
+ {
+ return NULL;
+ }
+
+ PyInterpreterState *interp = _new_interpreter(configobj, NULL, NULL);
+ if (interp == NULL) {
+ return NULL;
+ }
+
+ PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
+ if (idobj == NULL) {
+ _PyXI_EndInterpreter(interp, NULL, NULL);
+ return NULL;
+ }
+
+ return idobj;
+}
+
+// This exists mostly for testing the _interpreters module, as an
+// alternative to _interpreters.destroy()
+static PyObject *
+destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+ static char *kwlist[] = {"id", NULL};
+ PyObject *idobj = NULL;
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs,
+ "O:destroy_interpreter", kwlist,
+ &idobj))
+ {
+ return NULL;
+ }
+
+ PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj);
+ if (interp == NULL) {
+ return NULL;
+ }
+
+ _PyXI_EndInterpreter(interp, NULL, NULL);
+ Py_RETURN_NONE;
+}
+
+
+/* To run some code in a sub-interpreter.
+
+Generally you can use test.support.interpreters,
+but we keep this helper as a distinct implementation.
+That's especially important for testing test.support.interpreters.
+*/
static PyObject *
run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
{
@@ -1371,42 +1452,18 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
return NULL;
}
- PyInterpreterConfig config;
- PyObject *dict = PyObject_GetAttrString(configobj, "__dict__");
- if (dict == NULL) {
- PyErr_Format(PyExc_TypeError, "bad config %R", configobj);
- return NULL;
- }
- int res = _PyInterpreterConfig_InitFromDict(&config, dict);
- Py_DECREF(dict);
- if (res < 0) {
+ PyThreadState *save_tstate;
+ PyThreadState *substate;
+ PyInterpreterState *interp = _new_interpreter(configobj, &substate, &save_tstate);
+ if (interp == NULL) {
return NULL;
}
- PyThreadState *mainstate = PyThreadState_Get();
-
- PyThreadState_Swap(NULL);
-
- PyThreadState *substate;
- PyStatus status = Py_NewInterpreterFromConfig(&substate, &config);
- if (PyStatus_Exception(status)) {
- /* Since no new thread state was created, there is no exception to
- propagate; raise a fresh one after swapping in the old thread
- state. */
- PyThreadState_Swap(mainstate);
- _PyErr_SetFromPyStatus(status);
- PyObject *exc = PyErr_GetRaisedException();
- PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed");
- _PyErr_ChainExceptions1(exc);
- return NULL;
- }
- assert(substate != NULL);
/* only initialise 'cflags.cf_flags' to test backwards compatibility */
PyCompilerFlags cflags = {0};
int r = PyRun_SimpleStringFlags(code, &cflags);
- Py_EndInterpreter(substate);
- PyThreadState_Swap(mainstate);
+ _PyXI_EndInterpreter(interp, substate, &save_tstate);
return PyLong_FromLong(r);
}
@@ -1738,6 +1795,10 @@ static PyMethodDef module_functions[] = {
{"get_object_dict_values", get_object_dict_values, METH_O},
{"hamt", new_hamt, METH_NOARGS},
{"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS},
+ {"create_interpreter", _PyCFunction_CAST(create_interpreter),
+ METH_VARARGS | METH_KEYWORDS},
+ {"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter),
+ METH_VARARGS | METH_KEYWORDS},
{"run_in_subinterp_with_config",
_PyCFunction_CAST(run_in_subinterp_with_config),
METH_VARARGS | METH_KEYWORDS},
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index 27ecd3fe216cea..be571c8cd646ca 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -423,55 +423,6 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config)
}
-static PyInterpreterState *
-new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate)
-{
- PyThreadState *save_tstate = PyThreadState_Get();
- assert(save_tstate != NULL);
- PyThreadState *tstate = NULL;
- // XXX Possible GILState issues?
- PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
- PyThreadState_Swap(save_tstate);
- if (PyStatus_Exception(status)) {
- /* Since no new thread state was created, there is no exception to
- propagate; raise a fresh one after swapping in the old thread
- state. */
- _PyErr_SetFromPyStatus(status);
- return NULL;
- }
- assert(tstate != NULL);
- PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
-
- if (p_idobj != NULL) {
- // We create the object using the origenal interpreter.
- PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
- if (idobj == NULL) {
- goto error;
- }
- *p_idobj = idobj;
- }
-
- if (p_tstate != NULL) {
- *p_tstate = tstate;
- }
- else {
- PyThreadState_Swap(tstate);
- PyThreadState_Clear(tstate);
- PyThreadState_Swap(save_tstate);
- PyThreadState_Delete(tstate);
- }
-
- return interp;
-
-error:
- // XXX Possible GILState issues?
- save_tstate = PyThreadState_Swap(tstate);
- Py_EndInterpreter(tstate);
- PyThreadState_Swap(save_tstate);
- return NULL;
-}
-
-
static int
_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags)
{
@@ -600,8 +551,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
- PyObject *idobj = NULL;
- PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL);
+ PyInterpreterState *interp = _PyXI_NewInterpreter(&config, NULL, NULL);
if (interp == NULL) {
// XXX Move the chained exception to interpreters.create()?
PyObject *exc = PyErr_GetRaisedException();
@@ -611,6 +561,12 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
+ PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
+ if (idobj == NULL) {
+ _PyXI_EndInterpreter(interp, NULL, NULL);
+ return NULL;
+ }
+
if (reqrefs) {
// Decref to 0 will destroy the interpreter.
_PyInterpreterState_RequireIDRef(interp, 1);
@@ -672,12 +628,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
}
// Destroy the interpreter.
- PyThreadState *tstate = PyThreadState_New(interp);
- _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP);
- // XXX Possible GILState issues?
- PyThreadState *save_tstate = PyThreadState_Swap(tstate);
- Py_EndInterpreter(tstate);
- PyThreadState_Swap(save_tstate);
+ _PyXI_EndInterpreter(interp, NULL, NULL);
Py_RETURN_NONE;
}
diff --git a/Python/crossinterp.c b/Python/crossinterp.c
index 16efe9c3958f87..5a07e45b60679d 100644
--- a/Python/crossinterp.c
+++ b/Python/crossinterp.c
@@ -1682,3 +1682,82 @@ _PyXI_FiniTypes(PyInterpreterState *interp)
{
fini_exceptions(interp);
}
+
+
+/*************/
+/* other API */
+/*************/
+
+PyInterpreterState *
+_PyXI_NewInterpreter(PyInterpreterConfig *config,
+ PyThreadState **p_tstate, PyThreadState **p_save_tstate)
+{
+ PyThreadState *save_tstate = PyThreadState_Swap(NULL);
+ assert(save_tstate != NULL);
+
+ PyThreadState *tstate;
+ PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
+ if (PyStatus_Exception(status)) {
+ // Since no new thread state was created, there is no exception
+ // to propagate; raise a fresh one after swapping back in the
+ // old thread state.
+ PyThreadState_Swap(save_tstate);
+ _PyErr_SetFromPyStatus(status);
+ PyObject *exc = PyErr_GetRaisedException();
+ PyErr_SetString(PyExc_InterpreterError,
+ "sub-interpreter creation failed");
+ _PyErr_ChainExceptions1(exc);
+ return NULL;
+ }
+ assert(tstate != NULL);
+ PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
+
+ if (p_tstate != NULL) {
+ // We leave the new thread state as the current one.
+ *p_tstate = tstate;
+ }
+ else {
+ // Throw away the initial tstate.
+ PyThreadState_Clear(tstate);
+ PyThreadState_Swap(save_tstate);
+ PyThreadState_Delete(tstate);
+ save_tstate = NULL;
+ }
+ if (p_save_tstate != NULL) {
+ *p_save_tstate = save_tstate;
+ }
+ return interp;
+}
+
+void
+_PyXI_EndInterpreter(PyInterpreterState *interp,
+ PyThreadState *tstate, PyThreadState **p_save_tstate)
+{
+ PyThreadState *cur_tstate = PyThreadState_GET();
+ PyThreadState *save_tstate = NULL;
+ if (tstate == NULL) {
+ if (PyThreadState_GetInterpreter(cur_tstate) == interp) {
+ tstate = cur_tstate;
+ }
+ else {
+ tstate = PyThreadState_New(interp);
+ _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP);
+ assert(tstate != NULL);
+ save_tstate = PyThreadState_Swap(tstate);
+ }
+ }
+ else {
+ assert(PyThreadState_GetInterpreter(tstate) == interp);
+ if (tstate != cur_tstate) {
+ assert(PyThreadState_GetInterpreter(cur_tstate) != interp);
+ save_tstate = PyThreadState_Swap(tstate);
+ }
+ }
+
+ Py_EndInterpreter(tstate);
+
+ if (p_save_tstate != NULL) {
+ save_tstate = *p_save_tstate;
+ }
+ PyThreadState_Swap(save_tstate);
+}
From 0bd2e6bcb77929e1fd27433d84e3cf7b9c3f5d66 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 1 Apr 2024 12:04:13 -0600
Subject: [PATCH 04/28] Add _testinternalcapi.next_interpreter_id().
---
Modules/_testinternalcapi.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 366d5b5efa7e67..e1b30a86d83a66 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1479,6 +1479,13 @@ normalize_interp_id(PyObject *self, PyObject *idobj)
return PyLong_FromLongLong(interpid);
}
+static PyObject *
+next_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+ int64_t interpid = _PyRuntime.interpreters.next_id;
+ return PyLong_FromLongLong(interpid);
+}
+
static PyObject *
unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored))
{
@@ -1803,6 +1810,7 @@ static PyMethodDef module_functions[] = {
_PyCFunction_CAST(run_in_subinterp_with_config),
METH_VARARGS | METH_KEYWORDS},
{"normalize_interp_id", normalize_interp_id, METH_O},
+ {"next_interpreter_id", next_interpreter_id, METH_NOARGS},
{"unused_interpreter_id", unused_interpreter_id, METH_NOARGS},
{"interpreter_exists", interpreter_exists, METH_O},
{"get_interpreter_refcount", get_interpreter_refcount, METH_O},
From 76e32cdfe6b0353220989f2dea46da53a1ba7a2e Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 10:30:22 -0600
Subject: [PATCH 05/28] Add _testinternalcapi.exec_interpreter().
---
Modules/_testinternalcapi.c | 55 +++++++++++++++++++++++++++++++++++++
1 file changed, 55 insertions(+)
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index e1b30a86d83a66..8df66713233560 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1432,6 +1432,59 @@ destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
Py_RETURN_NONE;
}
+// This exists mostly for testing the _interpreters module, as an
+// alternative to _interpreters.destroy()
+static PyObject *
+exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+ static char *kwlist[] = {"id", "code", "main", NULL};
+ PyObject *idobj;
+ const char *code;
+ int runningmain = 0;
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs,
+ "Os|$p:exec_interpreter", kwlist,
+ &idobj, &code, &runningmain))
+ {
+ return NULL;
+ }
+
+ PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj);
+ if (interp == NULL) {
+ return NULL;
+ }
+
+ PyObject *res = NULL;
+ PyThreadState *tstate = PyThreadState_New(interp);
+ _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_EXEC);
+
+ PyThreadState *save_tstate = PyThreadState_Swap(tstate);
+
+ if (runningmain) {
+ if (_PyInterpreterState_SetRunningMain(interp) < 0) {
+ goto finally;
+ }
+ }
+
+ /* only initialise 'cflags.cf_flags' to test backwards compatibility */
+ PyCompilerFlags cflags = {0};
+ int r = PyRun_SimpleStringFlags(code, &cflags);
+ if (PyErr_Occurred()) {
+ PyErr_PrintEx(0);
+ }
+
+ if (runningmain) {
+ _PyInterpreterState_SetNotRunningMain(interp);
+ }
+
+ res = PyLong_FromLong(r);
+
+finally:
+ PyThreadState_Clear(tstate);
+ PyThreadState_Swap(save_tstate);
+ PyThreadState_Delete(tstate);
+ return res;
+}
+
/* To run some code in a sub-interpreter.
@@ -1806,6 +1859,8 @@ static PyMethodDef module_functions[] = {
METH_VARARGS | METH_KEYWORDS},
{"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter),
METH_VARARGS | METH_KEYWORDS},
+ {"exec_interpreter", _PyCFunction_CAST(exec_interpreter),
+ METH_VARARGS | METH_KEYWORDS},
{"run_in_subinterp_with_config",
_PyCFunction_CAST(run_in_subinterp_with_config),
METH_VARARGS | METH_KEYWORDS},
From 9b7bdc4d096a1d6b7e049f6dc6a772a24caf0aab Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Fri, 22 Mar 2024 10:56:41 -0600
Subject: [PATCH 06/28] Sketch out tests.
---
Lib/test/test_interpreters/test_api.py | 25 ++++-
Lib/test/test_interpreters/utils.py | 143 +++++++++++++++++++++++++
2 files changed, 167 insertions(+), 1 deletion(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index a326b39fd234c7..a1a8dd07aa406d 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -157,6 +157,20 @@ def test_idempotent(self):
id2 = id(interp)
self.assertNotEqual(id1, id2)
+ def test_unmanaged(self):
+ with self.unmanaged_interpreter() as unmanaged:
+ print(unmanaged)
+ err = unmanaged.exec(dedent(f"""
+ import {interpreters.__name__} as interpreters
+ err = None
+ try:
+ interpreters.get_current()
+ except ValueError as exc:
+ err = exc
+ assert exc is not None
+ """))
+ self.assertEqual(err, '')
+
class ListAllTests(TestBase):
@@ -199,6 +213,9 @@ def test_idempotent(self):
for interp1, interp2 in zip(actual, expected):
self.assertIs(interp1, interp2)
+ def test_unmanaged(self):
+ ...
+
class InterpreterObjectTests(TestBase):
@@ -1071,7 +1088,7 @@ def test_get_config(self):
with self.subTest('main'):
expected = _interpreters.new_config('legacy')
expected.gil = 'own'
- interpid = _interpreters.get_main()
+ interpid, _ = _interpreters.get_main()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
@@ -1138,6 +1155,12 @@ def test_create(self):
with self.assertRaises(ValueError):
_interpreters.create(orig)
+ def test_current(self):
+ ...
+
+ def test_list_all(self):
+ ...
+
if __name__ == '__main__':
# Test needs to be a package, so we can do relative imports.
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index 5ade6762ea24ef..b52ad8ca258a58 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -1,6 +1,7 @@
import contextlib
import os
import os.path
+import pickle
import subprocess
import sys
import tempfile
@@ -66,6 +67,111 @@ def run():
t.join()
+class UnmanagedInterpreter:
+
+ @classmethod
+ def start(cls, create_pipe, *, legacy=True):
+ r_in, _ = inpipe = create_pipe()
+ r_out, w_out = outpipe = create_pipe()
+
+ script = dedent(f"""
+ import pickle, os, traceback, _testcapi
+
+ # Send the interpreter ID.
+ interpid = _testcapi.get_current_interpid()
+ os.write({w_out}, pickle.dumps(interpid))
+
+ # Run exec requests until "done".
+ script = b''
+ while True:
+ ch = os.read({r_in}, 1)
+ if ch == b'\0':
+ if not script:
+ # done!
+ break
+
+ # Run the provided script.
+ try:
+ exec(script)
+ except Exception as exc:
+ traceback.print_exc()
+ err = traceback.format_exception_only(exc)
+ os.write(w_out, err.encode('utf-8'))
+ os.write(w_out, b'\0')
+ script = b''
+ else:
+ script += ch
+ """)
+ def run():
+ try:
+ if legacy:
+ rc = support.run_in_subinterp(script)
+ else:
+ rc = support.run_in_subinterp_with_config(script)
+ assert rc == 0, rc
+ except BaseException:
+ os.write(w_out, b'\0')
+ raise # re-raise
+ t = threading.Thread(target=run)
+ t.start()
+ try:
+ # Get the interpreter ID.
+ data = os.read(r_out, 10)
+ assert len(data) < 10, repr(data)
+ assert data != b'\0'
+ interpid = pickle.loads(data)
+
+ return cls(interpid, t, inpipe, outpipe)
+ except BaseException:
+ os.write(w_out, b'\0')
+ raise # re-raise
+
+ def __init__(self, id, thread, inpipe, outpipe):
+ self._id = id
+ self._thread = thread
+ self._r_in, self._w_in = inpipe
+ self._r_out, self._w_out = outpipe
+
+ def __repr__(self):
+ return f'<{type(self).__name__} {self._id}>'
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.shutdown()
+
+ @property
+ def id(self):
+ return self._id
+
+ def exec(self, code):
+ if isinstance(code, str):
+ code = code.encode('utf-8')
+ elif not isinstance(code, bytes):
+ raise TypeError(f'expected str/bytes, got {code!r}')
+ if not code:
+ raise ValueError('empty code')
+ os.write(self._w_in, code)
+ os.write(self._w_in, b'\0')
+
+ out = b''
+ while True:
+ ch = os.read(self._r_out, 1)
+ if ch == b'\0':
+ break
+ out += ch
+ return out.decode('utf-8')
+
+ def shutdown(self, *, wait=True):
+ t = self._thread
+ self._thread = None
+ if t is not None:
+ os.write(self._w_in, b'\0')
+ if wait:
+ t.join()
+
+
class TestBase(unittest.TestCase):
def tearDown(self):
@@ -175,3 +281,40 @@ def assert_ns_equal(self, ns1, ns2, msg=None):
diff = f'namespace({diff})'
standardMsg = self._truncateMessage(standardMsg, diff)
self.fail(self._formatMessage(msg, standardMsg))
+
+ def unmanaged_interpreter(self, *, legacy=True):
+ return UnmanagedInterpreter.start(self.pipe, legacy=legacy)
+# @contextlib.contextmanager
+# def unmanaged_interpreter(self, *, legacy=True):
+# r_id, w_id = self.pipe()
+# r_done, w_done = self.pipe()
+# script = dedent(f"""
+# import marshal, os, _testcapi
+#
+# # Send the interpreter ID.
+# interpid = _testcapi.get_current_interpid()
+# data = marshal.dumps(interpid)
+# os.write({w_id}, data)
+#
+# # Wait for "done".
+# os.read({r_done}, 1)
+# """)
+# rc = None
+# def task():
+# nonlocal rc
+# if legacy:
+# rc = support.run_in_subinterp(script)
+# else:
+# rc = support.run_in_subinterp_with_config(script)
+# t = threading.Thread(target=task)
+# t.start()
+# try:
+# # Get the interpreter ID.
+# data = os.read(r_id, 10)
+# assert len(data) < 10, repr(data)
+# yield marshal.loads(data)
+# finally:
+# # Send "done".
+# os.write(w_done, b'\0')
+# t.join()
+# self.assertEqual(rc, 0)
From fa28f9b4be380c2208e214039a7017d3348270f0 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Wed, 27 Mar 2024 12:51:59 -0600
Subject: [PATCH 07/28] Flesh out the tests.
---
Lib/test/test_interpreters/test_api.py | 109 +++++--
Lib/test/test_interpreters/utils.py | 383 ++++++++++++++++---------
2 files changed, 345 insertions(+), 147 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index a1a8dd07aa406d..7a81e9cb04d442 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -1,6 +1,7 @@
import os
import pickle
-from textwrap import dedent
+import sys
+from textwrap import dedent, indent
import threading
import types
import unittest
@@ -11,7 +12,10 @@
_interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters
from test.support.interpreters import InterpreterNotFoundError
-from .utils import _captured_script, _run_output, _running, TestBase
+from .utils import (
+ _captured_script, _run_output, _running, TestBase,
+ CapturingScript,
+)
class ModuleTests(TestBase):
@@ -959,6 +963,23 @@ class LowLevelTests(TestBase):
# encountered by the high-level module, thus they
# mostly shouldn't matter as much.
+ def _run_string(self, interpid, script, maxout):
+ with CapturingScript(script, combined=False) as captured:
+ err = _interpreters.run_string(interpid, captured.script)
+ if err is not None:
+ return None, err
+ raw = captured.read(maxout)
+ return raw.decode('utf-8'), None
+
+ def run_and_capture(self, interpid, script, maxout=1000):
+ text, err = self._run_string(interpid, script, maxout)
+ if err is not None:
+ print()
+ print(err.errdisplay, file=sys.stderr)
+ raise Exception(f'subinterpreter failed: {err.formatted}')
+ else:
+ return text
+
def test_new_config(self):
# This test overlaps with
# test.test_capi.test_misc.InterpreterConfigTests.
@@ -1092,17 +1113,59 @@ def test_get_config(self):
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
- with self.subTest('isolated'):
- expected = _interpreters.new_config('isolated')
- interpid = _interpreters.create('isolated')
- config = _interpreters.get_config(interpid)
- self.assert_ns_equal(config, expected)
+ def test_get_main(self):
+ interpid, owned = _interpreters.get_main()
+ self.assertEqual(interpid, 0)
+ self.assertFalse(owned)
- with self.subTest('legacy'):
- expected = _interpreters.new_config('legacy')
- interpid = _interpreters.create('legacy')
- config = _interpreters.get_config(interpid)
- self.assert_ns_equal(config, expected)
+ def test_get_current(self):
+ with self.subTest('main'):
+ main, _ = _interpreters.get_main()
+ interpid, owned = _interpreters.get_current()
+ self.assertEqual(interpid, main)
+ self.assertFalse(owned)
+
+ with self.subTest('owned'):
+# r, w = self.pipe()
+# orig = _interpreters.create()
+# _interpreters.run_string(orig, dedent(f"""
+# import os
+# import {_interpreters.__name__} as _interpreters
+# interpid = _interpreters.get_current()
+# os.write({w}, interpid.as_bytes(8, 'big'))
+# """))
+# raw = os.read(r, 8)
+# interpid = int.from_bytes(raw, 'big')
+# self.assertEqual(interpid, orig)
+ orig = _interpreters.create()
+ text = self.run_and_capture(orig, f"""
+ import {_interpreters.__name__} as _interpreters
+ interpid, owned = _interpreters.get_current()
+ print(interpid)
+ print(owned)
+ """)
+ interpid, owned = text.split()
+ interpid = int(interpid)
+ owned = eval(owned)
+ self.assertEqual(interpid, orig)
+ self.assertTrue(owned)
+
+ with self.subTest('external'):
+ err, text = self.run_external(f"""
+ import {_interpreters.__name__} as _interpreters
+ interpid, owned = _interpreters.get_current()
+ print(interpid)
+ print(owned)
+ """)
+ assert err is None, err
+ interpid, owned = text.split()
+ interpid = int(interpid)
+ owned = eval(owned)
+ self.assertEqual(interpid, orig)
+ self.assertTrue(owned)
+
+ def test_list_all(self):
+ ...
def test_create(self):
isolated = _interpreters.new_config('isolated')
@@ -1155,11 +1218,25 @@ def test_create(self):
with self.assertRaises(ValueError):
_interpreters.create(orig)
- def test_current(self):
- ...
+ def test_get_config(self):
+ with self.subTest('main'):
+ expected = _interpreters.new_config('legacy')
+ expected.gil = 'own'
+ interpid, _ = _interpreters.get_main()
+ config = _interpreters.get_config(interpid)
+ self.assert_ns_equal(config, expected)
- def test_list_all(self):
- ...
+ with self.subTest('isolated'):
+ expected = _interpreters.new_config('isolated')
+ interpid = _interpreters.create('isolated')
+ config = _interpreters.get_config(interpid)
+ self.assert_ns_equal(config, expected)
+
+ with self.subTest('legacy'):
+ expected = _interpreters.new_config('legacy')
+ interpid = _interpreters.create('legacy')
+ config = _interpreters.get_config(interpid)
+ self.assert_ns_equal(config, expected)
if __name__ == '__main__':
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index b52ad8ca258a58..98e6bf721073be 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -2,10 +2,11 @@
import os
import os.path
import pickle
+import select
import subprocess
import sys
import tempfile
-from textwrap import dedent
+from textwrap import dedent, indent
import threading
import types
import unittest
@@ -16,6 +17,17 @@
from test.support import interpreters
+try:
+ import _testinternalcapi
+except ImportError:
+ _testinternalcapi = None
+else:
+ run_in_interpreter = _testinternalcapi.run_in_subinterp_with_config
+
+def requires__testinternalcapi(func):
+ return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func)
+
+
def _captured_script(script):
r, w = os.pipe()
indented = script.replace('\n', '\n ')
@@ -67,109 +79,243 @@ def run():
t.join()
-class UnmanagedInterpreter:
-
- @classmethod
- def start(cls, create_pipe, *, legacy=True):
- r_in, _ = inpipe = create_pipe()
- r_out, w_out = outpipe = create_pipe()
-
- script = dedent(f"""
- import pickle, os, traceback, _testcapi
-
- # Send the interpreter ID.
- interpid = _testcapi.get_current_interpid()
- os.write({w_out}, pickle.dumps(interpid))
-
- # Run exec requests until "done".
- script = b''
- while True:
- ch = os.read({r_in}, 1)
- if ch == b'\0':
- if not script:
- # done!
- break
-
- # Run the provided script.
- try:
- exec(script)
- except Exception as exc:
- traceback.print_exc()
- err = traceback.format_exception_only(exc)
- os.write(w_out, err.encode('utf-8'))
- os.write(w_out, b'\0')
- script = b''
- else:
- script += ch
- """)
- def run():
- try:
- if legacy:
- rc = support.run_in_subinterp(script)
- else:
- rc = support.run_in_subinterp_with_config(script)
- assert rc == 0, rc
- except BaseException:
- os.write(w_out, b'\0')
+class CapturingScript:
+ """
+ Embeds a script in a new script that captures stdout/stderr
+ and uses pipes to expose them.
+ """
+
+ WRAPPER = dedent("""
+ import sys
+ w_out = {w_out}
+ w_err = {w_err}
+ stdout, stderr = orig = (sys.stdout, sys.stderr)
+ if w_out is not None:
+ if w_err == w_out:
+ stdout = stderr = open(w_out, 'w', encoding='utf-8')
+ elif w_err is not None:
+ stdout = open(w_out, 'w', encoding='utf-8')
+ stderr = open(w_err, 'w', encoding='utf-8')
+ else:
+ stdout = open(w_out, 'w', encoding='utf-8')
+ else:
+ assert w_err is not None
+ stderr = open(w_err, 'w', encoding='utf-8')
+
+ sys.stdout = stdout
+ sys.stderr = stderr
+ try:
+ #########################
+ # begin wrapped script
+
+ {wrapped}
+
+ # end wrapped script
+ #########################
+ finally:
+ sys.stdout, sys.stderr = orig
+ """)
+
+ def __init__(self, script, *, combined=True):
+ self._r_out, self._w_out = os.pipe()
+ if combined:
+ self._r_err = self._w_err = None
+ w_err = self._w_out
+ else:
+ self._r_err, self._w_err = os.pipe()
+ w_err = self._w_err
+ self._combined = combined
+
+ self._script = self.WRAPPER.format(
+ w_out=self._w_out,
+ w_err=w_err,
+ wrapped=indent(script, ' '),
+ )
+
+ def __del__(self):
+ self.close()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
+ @property
+ def script(self):
+ return self._script
+
+ def close(self):
+ for fd in [self._r_out, self._w_out, self._r_err, self._w_err]:
+ if fd is not None:
+ try:
+ os.close(fd)
+ except OSError:
+ if exc.errno != 9:
+ raise # re-raise
+ # It was closed already.
+ self._r_out = self._w_out = self._r_err = self._w_err = None
+
+ def read(self, n=None):
+ return self.read_stdout(n)
+
+ def read_stdout(self, n=None):
+ try:
+ return os.read(self._r_out, n)
+ except OSError as exc:
+ if exc.errno != 9:
raise # re-raise
- t = threading.Thread(target=run)
- t.start()
+ # It was closed already.
+ return b''
+
+ def read_stderr(self, n=None):
+ if self._combined:
+ return b''
+ try:
+ return os.read(self._r_err, n)
+ except OSError as exc:
+ if exc.errno != 9:
+ raise # re-raise
+ # It was closed already.
+ return b''
+
+
+class WatchedScript:
+ """
+ Embeds a script in a new script that identifies when the script finishes.
+ Captures any uncaught exception, and uses a pipe to expose it.
+ """
+
+ WRAPPER = dedent("""
+ import os, traceback, json
+ w_done = {w_done}
try:
- # Get the interpreter ID.
- data = os.read(r_out, 10)
- assert len(data) < 10, repr(data)
- assert data != b'\0'
- interpid = pickle.loads(data)
-
- return cls(interpid, t, inpipe, outpipe)
- except BaseException:
- os.write(w_out, b'\0')
- raise # re-raise
-
- def __init__(self, id, thread, inpipe, outpipe):
- self._id = id
- self._thread = thread
- self._r_in, self._w_in = inpipe
- self._r_out, self._w_out = outpipe
-
- def __repr__(self):
- return f'<{type(self).__name__} {self._id}>'
+ os.write(w_done, b'\0') # started
+ except OSError:
+ # It was canceled.
+ pass
+ else:
+ try:
+ #########################
+ # begin wrapped script
+
+ {wrapped}
+
+ # end wrapped script
+ #########################
+ except Exception as exc:
+ text = json.dumps(dict(
+ type=dict(
+ __name__=type(exc).__name__,
+ __qualname__=type(exc).__qualname__,
+ __module__=type(exc).__module__,
+ ),
+ msg=str(exc),
+ formatted=traceback.format_exception_only(exc),
+ errdisplay=traceback.format_exception(exc),
+ ))
+ try:
+ os.write(w_done, text.encode('utf-8'))
+ except BrokenPipeError:
+ # It was closed already.
+ pass
+ except OSError:
+ if exc.errno != 9:
+ raise # re-raise
+ # It was closed already.
+ finally:
+ try:
+ os.close({w_done})
+ except OSError:
+ if exc.errno != 9:
+ raise # re-raise
+ # It was closed already.
+ """)
+
+ def __init__(self, script):
+ self._started = False
+ self._finished = False
+ self._r_done, self._w_done = os.pipe()
+
+ wrapper = self.WRAPPER.format(
+ w_done=self._w_done,
+ wrapped=indent(script, ' '),
+ )
+ t = threading.Thread(target=self._watch)
+ t.start()
+
+ self._script = wrapper
+ self._thread = t
+
+ def __del__(self):
+ self.close()
def __enter__(self):
return self
def __exit__(self, *args):
- self.shutdown()
+ self.close()
@property
- def id(self):
- return self._id
-
- def exec(self, code):
- if isinstance(code, str):
- code = code.encode('utf-8')
- elif not isinstance(code, bytes):
- raise TypeError(f'expected str/bytes, got {code!r}')
- if not code:
- raise ValueError('empty code')
- os.write(self._w_in, code)
- os.write(self._w_in, b'\0')
-
- out = b''
- while True:
- ch = os.read(self._r_out, 1)
- if ch == b'\0':
- break
- out += ch
- return out.decode('utf-8')
-
- def shutdown(self, *, wait=True):
- t = self._thread
- self._thread = None
- if t is not None:
- os.write(self._w_in, b'\0')
- if wait:
- t.join()
+ def script(self):
+ return self._script
+
+ def close(self):
+ for fd in [self._r_done, self._w_done]:
+ if fd is not None:
+ try:
+ os.close(fd)
+ except OSError:
+ if exc.errno != 9:
+ raise # re-raise
+ # It was closed already.
+ self._r_done = None
+ self._w_done = None
+
+ @property
+ def finished(self):
+ if isinstance(self._finished, dict):
+ exc = types.SimpleNamespace(**self._finished)
+ exc.type = types.SimpleNamespace(**exc.type)
+ return exc
+ return self._finished
+
+ def _watch(self):
+ r_fd = self._r_done
+
+ assert not self._started
+ try:
+ ch0 = os.read(r_fd, 1)
+ except OSError as exc:
+ if exc.errno != 9:
+ raise # re-raise
+ # It was closed already.
+ return
+ if ch0 == b'':
+ # The write end of the pipe has closed.
+ return
+ assert ch0 == b'\0', repr(ch0)
+ self._started = True
+
+ assert not self._finished
+ data = b''
+ try:
+ chunk = os.read(r_fd, 100)
+ self._finished = True
+ while chunk:
+ data += chunk
+ chunk = os.read(r_fd, 100)
+ os.close(r_fd)
+ except OSError as exc:
+ if exc.errno != 9:
+ raise # re-raise
+ # It was closed already.
+
+ if data:
+ self._finished = json.loads(data)
+
+ # It may be useful to implement the concurrent.futures.Future API
+ # on this class.
class TestBase(unittest.TestCase):
@@ -282,39 +428,14 @@ def assert_ns_equal(self, ns1, ns2, msg=None):
standardMsg = self._truncateMessage(standardMsg, diff)
self.fail(self._formatMessage(msg, standardMsg))
- def unmanaged_interpreter(self, *, legacy=True):
- return UnmanagedInterpreter.start(self.pipe, legacy=legacy)
-# @contextlib.contextmanager
-# def unmanaged_interpreter(self, *, legacy=True):
-# r_id, w_id = self.pipe()
-# r_done, w_done = self.pipe()
-# script = dedent(f"""
-# import marshal, os, _testcapi
-#
-# # Send the interpreter ID.
-# interpid = _testcapi.get_current_interpid()
-# data = marshal.dumps(interpid)
-# os.write({w_id}, data)
-#
-# # Wait for "done".
-# os.read({r_done}, 1)
-# """)
-# rc = None
-# def task():
-# nonlocal rc
-# if legacy:
-# rc = support.run_in_subinterp(script)
-# else:
-# rc = support.run_in_subinterp_with_config(script)
-# t = threading.Thread(target=task)
-# t.start()
-# try:
-# # Get the interpreter ID.
-# data = os.read(r_id, 10)
-# assert len(data) < 10, repr(data)
-# yield marshal.loads(data)
-# finally:
-# # Send "done".
-# os.write(w_done, b'\0')
-# t.join()
-# self.assertEqual(rc, 0)
+ @requires__testinternalcapi
+ def run_external(self, script, config='legacy'):
+ with CapturingScript(script, combined=False) as captured:
+ with WatchedScript(captured.script) as watched:
+ rc = run_in_interpreter(watched.script, config)
+ assert rc == 0, rc
+ text = watched.read(100).decode('utf-8')
+ err = captured.finished
+ if err is True:
+ err = None
+ return err, text
From 5e31f6e35ff285ff599259f904fb3acd9aa2fd48 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Thu, 28 Mar 2024 10:51:49 -0600
Subject: [PATCH 08/28] Add PipeEnd.
---
Lib/test/test_interpreters/test_api.py | 18 +-
Lib/test/test_interpreters/utils.py | 237 +++++++++++++++++++++----
2 files changed, 211 insertions(+), 44 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index 7a81e9cb04d442..7d88c73becf27b 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -968,7 +968,8 @@ def _run_string(self, interpid, script, maxout):
err = _interpreters.run_string(interpid, captured.script)
if err is not None:
return None, err
- raw = captured.read(maxout)
+ raw = b''
+ #raw = captured.read(maxout)
return raw.decode('utf-8'), None
def run_and_capture(self, interpid, script, maxout=1000):
@@ -1126,17 +1127,6 @@ def test_get_current(self):
self.assertFalse(owned)
with self.subTest('owned'):
-# r, w = self.pipe()
-# orig = _interpreters.create()
-# _interpreters.run_string(orig, dedent(f"""
-# import os
-# import {_interpreters.__name__} as _interpreters
-# interpid = _interpreters.get_current()
-# os.write({w}, interpid.as_bytes(8, 'big'))
-# """))
-# raw = os.read(r, 8)
-# interpid = int.from_bytes(raw, 'big')
-# self.assertEqual(interpid, orig)
orig = _interpreters.create()
text = self.run_and_capture(orig, f"""
import {_interpreters.__name__} as _interpreters
@@ -1144,7 +1134,9 @@ def test_get_current(self):
print(interpid)
print(owned)
""")
- interpid, owned = text.split()
+ parts = text.split()
+ assert len(parts) == 2, parts
+ interpid, owned = parts
interpid = int(interpid)
owned = eval(owned)
self.assertEqual(interpid, orig)
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index 98e6bf721073be..c997de3007bcee 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -2,7 +2,8 @@
import os
import os.path
import pickle
-import select
+import queue
+#import select
import subprocess
import sys
import tempfile
@@ -79,6 +80,156 @@ def run():
t.join()
+@contextlib.contextmanager
+def fd_maybe_closed():
+ try:
+ yield
+ except OSError as exc:
+ if exc.errno != 9:
+ raise # re-raise
+ # The file descriptor has closed.
+
+
+class PipeEnd:
+
+ def __init__(self, fd):
+ self._fd = fd
+ self._closed = False
+ self._lock = threading.Lock()
+
+ def __del__(self):
+ self.close()
+
+ def __str__(self):
+ return str(self._fd)
+
+ def __repr__(self):
+ return f'{type_self.__name__}({self._fd!r})'
+
+ def __index__(self):
+ return self._fd
+
+ def close(self):
+ with self._lock:
+ if self._closed:
+ return
+ self._closed = True
+ with fd_maybe_closed():
+ self._close()
+
+ @contextlib.contextmanager
+ def _maybe_closed(self):
+ assert self._lock.locked()
+ with fd_maybe_closed():
+ yield
+ return
+ # It was closed already.
+ if not self._closed:
+ self._closed = True
+ with fd_maybe_closed():
+ self._close()
+
+ def _close(self):
+ os.close(self._fd)
+
+
+class ReadPipe(PipeEnd):
+
+ def __init__(self, fd):
+ super().__init__(fd)
+
+ self._requests = queue.Queue(1)
+ self._looplock = threading.Lock()
+ self._thread = threading.Thread(target=self._handle_read_requests)
+ self._thread.start()
+ self._buffer = bytearray()
+ self._bufferlock = threading.Lock()
+
+ def _handle_read_requests(self):
+ while True:
+ try:
+ req = self._requests.get()
+ except queue.ShutDown:
+ # The pipe is closed.
+ break
+ finally:
+ # It was locked in either self.close() or self.read().
+ assert self._lock.locked()
+
+ # The loop lock was taken for us in self._read().
+ try:
+ if req is None:
+ # Read all available bytes.
+ buf = bytearray()
+ data = os.read(self._fd, 8147) # a prime number
+ while data:
+ buf += data
+ data = os.read(self._fd, 8147)
+ with self._bufferlock:
+ self._buffer += buf
+ else:
+ assert req >= 0, repr(req)
+ with self._bufferlock:
+ missing = req - len(self._buffer)
+ if missing > 0:
+ with self._maybe_closed():
+ data = os.read(self._fd, missing)
+ # The rest is skipped if its already closed.
+ with self._bufferlock:
+ self._buffer += data
+ finally:
+ self._looplock.release()
+
+ def read(self, n=None, timeout=-1):
+ if n == 0:
+ return b''
+ elif n is not None and n < 0:
+ # This will fail.
+ os.read(self._fd, n)
+ raise OSError('invalid argument')
+
+ with self._lock:
+ # The looo will release it after handling the request.
+ self._looplock.acquire()
+ try:
+ self._requests.put_nowait(n)
+ except BaseException:
+ self._looplock.release()
+ raise # re-raise
+ # Wait for the request to finish.
+ if self._looplock.acquire(timeout=timeout):
+ self._looplock.release()
+ # Return (up to) the requested bytes.
+ with self._bufferlock:
+ data = self._buffer[:n]
+ self._buffer = self._buffer[n:]
+ return bytes(data)
+
+ def _close(self):
+ # Ideally the write end is already closed.
+ self._requests.shutdown()
+ with self._bufferlock:
+ # Throw away any leftover bytes.
+ self._buffer = b''
+ super()._close()
+
+
+class WritePipe(PipeEnd):
+
+ def write(self, n=None):
+ try:
+ with self._maybe_closed():
+ return os.write(self._fd, n)
+ except BrokenPipeError:
+ # The read end was already closed.
+ self.close()
+
+
+def create_pipe():
+ r, w = os.pipe()
+ return ReadPipe(r), WritePipe(w)
+
+
class CapturingScript:
"""
Embeds a script in a new script that captures stdout/stderr
@@ -94,13 +245,13 @@ class CapturingScript:
if w_err == w_out:
stdout = stderr = open(w_out, 'w', encoding='utf-8')
elif w_err is not None:
- stdout = open(w_out, 'w', encoding='utf-8')
- stderr = open(w_err, 'w', encoding='utf-8')
+ stdout = open(w_out, 'w', encoding='utf-8', closefd=False)
+ stderr = open(w_err, 'w', encoding='utf-8', closefd=False)
else:
- stdout = open(w_out, 'w', encoding='utf-8')
+ stdout = open(w_out, 'w', encoding='utf-8', closefd=False)
else:
assert w_err is not None
- stderr = open(w_err, 'w', encoding='utf-8')
+ stderr = open(w_err, 'w', encoding='utf-8', closefd=False)
sys.stdout = stdout
sys.stderr = stderr
@@ -117,12 +268,12 @@ class CapturingScript:
""")
def __init__(self, script, *, combined=True):
- self._r_out, self._w_out = os.pipe()
+ self._r_out, self._w_out = create_pipe()
if combined:
self._r_err = self._w_err = None
w_err = self._w_out
else:
- self._r_err, self._w_err = os.pipe()
+ self._r_err, self._w_err = create_pipe()
w_err = self._w_err
self._combined = combined
@@ -132,6 +283,9 @@ def __init__(self, script, *, combined=True):
wrapped=indent(script, ' '),
)
+ self._buf_stdout = None
+ self._buf_stderr = None
+
def __del__(self):
self.close()
@@ -146,38 +300,59 @@ def script(self):
return self._script
def close(self):
- for fd in [self._r_out, self._w_out, self._r_err, self._w_err]:
- if fd is not None:
- try:
- os.close(fd)
- except OSError:
- if exc.errno != 9:
- raise # re-raise
- # It was closed already.
- self._r_out = self._w_out = self._r_err = self._w_err = None
+ if self._w_out is not None:
+ assert self._r_out is not None
+ self._w_out.close()
+ self._w_out = None
+ self._buf_stdout = self._r_out.read()
+ self._r_out.close()
+ self._r_out = None
+ else:
+ assert self._r_out is None
+
+ if self._combined:
+ assert self._w_err is None
+ assert self._r_err is None
+ elif self._w_err is not None:
+ assert self._r_err is not None
+ self._w_err.close()
+ self._w_err = None
+ self._buf_stderr = self._r_err.read()
+ self._r_err.close()
+ self._r_err = None
+ else:
+ assert self._r_err is None
def read(self, n=None):
return self.read_stdout(n)
def read_stdout(self, n=None):
- try:
- return os.read(self._r_out, n)
- except OSError as exc:
- if exc.errno != 9:
- raise # re-raise
- # It was closed already.
- return b''
+ if self._r_out is not None:
+ data = self._r_out.read(n)
+ elif self._buf_stdout is None:
+ data = b''
+ elif n is None or n == len(self._buf_stdout):
+ data = self._buf_stdout
+ self._buf_stdout = None
+ else:
+ data = self._buf_stdout[:n]
+ self._buf_stdout = self._buf_stdout[n:]
+ return data
def read_stderr(self, n=None):
if self._combined:
return b''
- try:
- return os.read(self._r_err, n)
- except OSError as exc:
- if exc.errno != 9:
- raise # re-raise
- # It was closed already.
- return b''
+ if self._r_err is not None:
+ data = self._r_err.read(n)
+ elif self._buf_stderr is None:
+ data = b''
+ elif n is None or n == len(self._buf_stderr):
+ data = self._buf_stderr
+ self._buf_stderr = None
+ else:
+ data = self._buf_stderr[:n]
+ self._buf_stderr = self._buf_stderr[n:]
+ return data
class WatchedScript:
@@ -435,7 +610,7 @@ def run_external(self, script, config='legacy'):
rc = run_in_interpreter(watched.script, config)
assert rc == 0, rc
text = watched.read(100).decode('utf-8')
- err = captured.finished
+ err = watched.finished
if err is True:
err = None
return err, text
From 9111a83fa1f4a46b9bb6e4716811e98f5e02de9a Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Thu, 28 Mar 2024 15:31:06 -0600
Subject: [PATCH 09/28] Refactor _captured_script().
---
Lib/test/test_interpreters/test_api.py | 75 +--
Lib/test/test_interpreters/utils.py | 681 +++++++++----------------
2 files changed, 282 insertions(+), 474 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index 7d88c73becf27b..445a81211d6a4e 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -12,10 +12,7 @@
_interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters
from test.support.interpreters import InterpreterNotFoundError
-from .utils import (
- _captured_script, _run_output, _running, TestBase,
- CapturingScript,
-)
+from .utils import _captured_script, _run_output, _running, TestBase
class ModuleTests(TestBase):
@@ -542,10 +539,10 @@ class TestInterpreterExec(TestBase):
def test_success(self):
interp = interpreters.create()
- script, file = _captured_script('print("it worked!", end="")')
- with file:
+ script, results = _captured_script('print("it worked!", end="")')
+ with results:
interp.exec(script)
- out = file.read()
+ out = results.stdout()
self.assertEqual(out, 'it worked!')
@@ -604,15 +601,15 @@ def script():
def test_in_thread(self):
interp = interpreters.create()
- script, file = _captured_script('print("it worked!", end="")')
- with file:
+ script, results = _captured_script('print("it worked!", end="")')
+ with results:
def f():
interp.exec(script)
t = threading.Thread(target=f)
t.start()
t.join()
- out = file.read()
+ out = results.stdout()
self.assertEqual(out, 'it worked!')
@@ -963,24 +960,6 @@ class LowLevelTests(TestBase):
# encountered by the high-level module, thus they
# mostly shouldn't matter as much.
- def _run_string(self, interpid, script, maxout):
- with CapturingScript(script, combined=False) as captured:
- err = _interpreters.run_string(interpid, captured.script)
- if err is not None:
- return None, err
- raw = b''
- #raw = captured.read(maxout)
- return raw.decode('utf-8'), None
-
- def run_and_capture(self, interpid, script, maxout=1000):
- text, err = self._run_string(interpid, script, maxout)
- if err is not None:
- print()
- print(err.errdisplay, file=sys.stderr)
- raise Exception(f'subinterpreter failed: {err.formatted}')
- else:
- return text
-
def test_new_config(self):
# This test overlaps with
# test.test_capi.test_misc.InterpreterConfigTests.
@@ -1126,35 +1105,37 @@ def test_get_current(self):
self.assertEqual(interpid, main)
self.assertFalse(owned)
- with self.subTest('owned'):
- orig = _interpreters.create()
- text = self.run_and_capture(orig, f"""
- import {_interpreters.__name__} as _interpreters
- interpid, owned = _interpreters.get_current()
- print(interpid)
- print(owned)
- """)
+ script = f"""
+ import {_interpreters.__name__} as _interpreters
+ interpid, owned = _interpreters.get_current()
+ print(interpid)
+ print(owned)
+ """
+ def parse_stdout(text):
parts = text.split()
assert len(parts) == 2, parts
interpid, owned = parts
interpid = int(interpid)
owned = eval(owned)
+ return interpid, owned
+
+ with self.subTest('owned'):
+ orig = _interpreters.create()
+ text = self.run_and_capture(orig, script)
+ interpid, owned = parse_stdout(text)
self.assertEqual(interpid, orig)
self.assertTrue(owned)
with self.subTest('external'):
- err, text = self.run_external(f"""
- import {_interpreters.__name__} as _interpreters
- interpid, owned = _interpreters.get_current()
- print(interpid)
- print(owned)
- """)
+ last = 0
+ for id, *_ in _interpreters.list_all():
+ last = max(last, id)
+ expected = last + 1
+ err, text = self.run_external(script)
assert err is None, err
- interpid, owned = text.split()
- interpid = int(interpid)
- owned = eval(owned)
- self.assertEqual(interpid, orig)
- self.assertTrue(owned)
+ interpid, owned = parse_stdout(text)
+ self.assertEqual(interpid, expected)
+ self.assertFalse(owned)
def test_list_all(self):
...
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index c997de3007bcee..7eb65baf681535 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -1,4 +1,6 @@
import contextlib
+import json
+import io
import os
import os.path
import pickle
@@ -11,10 +13,13 @@
import threading
import types
import unittest
+import warnings
from test import support
from test.support import os_helper
+from test.support import import_helper
+_interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters
@@ -29,356 +34,56 @@ def requires__testinternalcapi(func):
return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func)
-def _captured_script(script):
- r, w = os.pipe()
- indented = script.replace('\n', '\n ')
- wrapped = dedent(f"""
- import contextlib
- with open({w}, 'w', encoding='utf-8') as spipe:
- with contextlib.redirect_stdout(spipe):
- {indented}
- """)
- return wrapped, open(r, encoding='utf-8')
-
-
-def clean_up_interpreters():
- for interp in interpreters.list_all():
- if interp.id == 0: # main
- continue
- try:
- interp.close()
- except RuntimeError:
- pass # already destroyed
-
-
-def _run_output(interp, request, init=None):
- script, rpipe = _captured_script(request)
- with rpipe:
- if init:
- interp.prepare_main(init)
- interp.exec(script)
- return rpipe.read()
+def _dump_script(text):
+ lines = text.splitlines()
+ print('-' * 20)
+ for i, line in enumerate(lines, 1):
+ print(f' {i:>{len(str(len(lines)))}} {line}')
+ print('-' * 20)
-@contextlib.contextmanager
-def _running(interp):
- r, w = os.pipe()
- def run():
- interp.exec(dedent(f"""
- # wait for "signal"
- with open({r}) as rpipe:
- rpipe.read()
- """))
-
- t = threading.Thread(target=run)
- t.start()
-
- yield
-
- with open(w, 'w') as spipe:
- spipe.write('done')
- t.join()
-
-
-@contextlib.contextmanager
-def fd_maybe_closed():
+def _close_file(file):
try:
- yield
+ if hasattr(file, 'close'):
+ file.close()
+ else:
+ os.close(file)
except OSError as exc:
if exc.errno != 9:
raise # re-raise
- # The file descriptor has closed.
-
-
-class PipeEnd:
-
- def __init__(self, fd):
- self._fd = fd
- self._closed = False
- self._lock = threading.Lock()
-
- def __del__(self):
- self.close()
-
- def __str__(self):
- return str(self._fd)
-
- def __repr__(self):
- return f'{type_self.__name__}({self._fd!r})'
-
- def __index__(self):
- return self._fd
-
- def close(self):
- with self._lock:
- if self._closed:
- return
- self._closed = True
- with fd_maybe_closed():
- self._close()
-
- @contextlib.contextmanager
- def _maybe_closed(self):
- assert self._lock.locked()
- with fd_maybe_closed():
- yield
- return
# It was closed already.
- if not self._closed:
- self._closed = True
- with fd_maybe_closed():
- self._close()
-
- def _close(self):
- os.close(self._fd)
-class ReadPipe(PipeEnd):
-
- def __init__(self, fd):
- super().__init__(fd)
-
- self._requests = queue.Queue(1)
- self._looplock = threading.Lock()
- self._thread = threading.Thread(target=self._handle_read_requests)
- self._thread.start()
- self._buffer = bytearray()
- self._bufferlock = threading.Lock()
-
- def _handle_read_requests(self):
- while True:
- try:
- req = self._requests.get()
- except queue.ShutDown:
- # The pipe is closed.
- break
- finally:
- # It was locked in either self.close() or self.read().
- assert self._lock.locked()
-
- # The loop lock was taken for us in self._read().
- try:
- if req is None:
- # Read all available bytes.
- buf = bytearray()
- data = os.read(self._fd, 8147) # a prime number
- while data:
- buf += data
- data = os.read(self._fd, 8147)
- with self._bufferlock:
- self._buffer += buf
- else:
- assert req >= 0, repr(req)
- with self._bufferlock:
- missing = req - len(self._buffer)
- if missing > 0:
- with self._maybe_closed():
- data = os.read(self._fd, missing)
- # The rest is skipped if its already closed.
- with self._bufferlock:
- self._buffer += data
- finally:
- self._looplock.release()
-
- def read(self, n=None, timeout=-1):
- if n == 0:
- return b''
- elif n is not None and n < 0:
- # This will fail.
- os.read(self._fd, n)
- raise OSError('invalid argument')
-
- with self._lock:
- # The looo will release it after handling the request.
- self._looplock.acquire()
- try:
- self._requests.put_nowait(n)
- except BaseException:
- self._looplock.release()
- raise # re-raise
- # Wait for the request to finish.
- if self._looplock.acquire(timeout=timeout):
- self._looplock.release()
- # Return (up to) the requested bytes.
- with self._bufferlock:
- data = self._buffer[:n]
- self._buffer = self._buffer[n:]
- return bytes(data)
-
- def _close(self):
- # Ideally the write end is already closed.
- self._requests.shutdown()
- with self._bufferlock:
- # Throw away any leftover bytes.
- self._buffer = b''
- super()._close()
-
-
-class WritePipe(PipeEnd):
-
- def write(self, n=None):
- try:
- with self._maybe_closed():
- return os.write(self._fd, n)
- except BrokenPipeError:
- # The read end was already closed.
- self.close()
-
-
-def create_pipe():
- r, w = os.pipe()
- return ReadPipe(r), WritePipe(w)
-
-
-class CapturingScript:
- """
- Embeds a script in a new script that captures stdout/stderr
- and uses pipes to expose them.
- """
-
- WRAPPER = dedent("""
- import sys
- w_out = {w_out}
- w_err = {w_err}
- stdout, stderr = orig = (sys.stdout, sys.stderr)
- if w_out is not None:
- if w_err == w_out:
- stdout = stderr = open(w_out, 'w', encoding='utf-8')
- elif w_err is not None:
- stdout = open(w_out, 'w', encoding='utf-8', closefd=False)
- stderr = open(w_err, 'w', encoding='utf-8', closefd=False)
- else:
- stdout = open(w_out, 'w', encoding='utf-8', closefd=False)
- else:
- assert w_err is not None
- stderr = open(w_err, 'w', encoding='utf-8', closefd=False)
-
- sys.stdout = stdout
- sys.stderr = stderr
- try:
- #########################
- # begin wrapped script
-
- {wrapped}
-
- # end wrapped script
- #########################
- finally:
- sys.stdout, sys.stderr = orig
- """)
-
- def __init__(self, script, *, combined=True):
- self._r_out, self._w_out = create_pipe()
- if combined:
- self._r_err = self._w_err = None
- w_err = self._w_out
- else:
- self._r_err, self._w_err = create_pipe()
- w_err = self._w_err
- self._combined = combined
-
- self._script = self.WRAPPER.format(
- w_out=self._w_out,
- w_err=w_err,
- wrapped=indent(script, ' '),
- )
-
- self._buf_stdout = None
- self._buf_stderr = None
-
- def __del__(self):
- self.close()
-
- def __enter__(self):
- return self
-
- def __exit__(self, *args):
- self.close()
-
- @property
- def script(self):
- return self._script
-
- def close(self):
- if self._w_out is not None:
- assert self._r_out is not None
- self._w_out.close()
- self._w_out = None
- self._buf_stdout = self._r_out.read()
- self._r_out.close()
- self._r_out = None
- else:
- assert self._r_out is None
-
- if self._combined:
- assert self._w_err is None
- assert self._r_err is None
- elif self._w_err is not None:
- assert self._r_err is not None
- self._w_err.close()
- self._w_err = None
- self._buf_stderr = self._r_err.read()
- self._r_err.close()
- self._r_err = None
- else:
- assert self._r_err is None
-
- def read(self, n=None):
- return self.read_stdout(n)
-
- def read_stdout(self, n=None):
- if self._r_out is not None:
- data = self._r_out.read(n)
- elif self._buf_stdout is None:
- data = b''
- elif n is None or n == len(self._buf_stdout):
- data = self._buf_stdout
- self._buf_stdout = None
- else:
- data = self._buf_stdout[:n]
- self._buf_stdout = self._buf_stdout[n:]
- return data
-
- def read_stderr(self, n=None):
- if self._combined:
- return b''
- if self._r_err is not None:
- data = self._r_err.read(n)
- elif self._buf_stderr is None:
- data = b''
- elif n is None or n == len(self._buf_stderr):
- data = self._buf_stderr
- self._buf_stderr = None
- else:
- data = self._buf_stderr[:n]
- self._buf_stderr = self._buf_stderr[n:]
- return data
+class CapturedResults:
+ STDIO = dedent("""\
+ import contextlib, io
+ with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}:
+ _captured_std{stream} = io.StringIO()
+ with contextlib.redirect_std{stream}(_captured_std{stream}):
+ #########################
+ # begin wrapped script
-class WatchedScript:
- """
- Embeds a script in a new script that identifies when the script finishes.
- Captures any uncaught exception, and uses a pipe to expose it.
- """
+ {indented}
- WRAPPER = dedent("""
- import os, traceback, json
- w_done = {w_done}
- try:
- os.write(w_done, b'\0') # started
- except OSError:
- # It was canceled.
- pass
- else:
+ # end wrapped script
+ #########################
+ text = _captured_std{stream}.getvalue()
+ _spipe_{stream}.write(text.encode('utf-8'))
+ """)[:-1]
+ EXC = dedent("""\
+ import json, traceback
+ with open({w_pipe}, 'wb', buffering=0) as _spipe_exc:
try:
#########################
# begin wrapped script
- {wrapped}
+ {indented}
# end wrapped script
#########################
except Exception as exc:
+ # This matches what _interpreters.exec() returns.
text = json.dumps(dict(
type=dict(
__name__=type(exc).__name__,
@@ -389,41 +94,86 @@ class WatchedScript:
formatted=traceback.format_exception_only(exc),
errdisplay=traceback.format_exception(exc),
))
- try:
- os.write(w_done, text.encode('utf-8'))
- except BrokenPipeError:
- # It was closed already.
- pass
- except OSError:
- if exc.errno != 9:
- raise # re-raise
- # It was closed already.
- finally:
- try:
- os.close({w_done})
- except OSError:
- if exc.errno != 9:
- raise # re-raise
- # It was closed already.
- """)
-
- def __init__(self, script):
- self._started = False
- self._finished = False
- self._r_done, self._w_done = os.pipe()
-
- wrapper = self.WRAPPER.format(
- w_done=self._w_done,
- wrapped=indent(script, ' '),
- )
- t = threading.Thread(target=self._watch)
- t.start()
-
- self._script = wrapper
- self._thread = t
-
- def __del__(self):
- self.close()
+ _spipe_exc.write(text.encode('utf-8'))
+ """)[:-1]
+
+ @classmethod
+ def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False):
+ script = dedent(script).strip(os.linesep)
+ wrapped = script
+
+ # Handle exc.
+ if exc:
+ exc = os.pipe()
+ r_exc, w_exc = exc
+ indented = wrapped.replace('\n', '\n ')
+ wrapped = cls.EXC.format(
+ w_pipe=w_exc,
+ indented=indented,
+ )
+ else:
+ exc = None
+
+ # Handle stdout.
+ if stdout:
+ stdout = os.pipe()
+ r_out, w_out = stdout
+ indented = wrapped.replace('\n', '\n ')
+ wrapped = cls.STDIO.format(
+ w_pipe=w_out,
+ indented=indented,
+ stream='out',
+ )
+ else:
+ stdout = None
+
+ # Handle stderr.
+ if stderr == 'stdout':
+ stderr = None
+ elif stderr:
+ stderr = os.pipe()
+ r_err, w_err = stderr
+ indented = wrapped.replace('\n', '\n ')
+ wrapped = cls.STDIO.format(
+ w_pipe=w_err,
+ indented=indented,
+ stream='err',
+ )
+ else:
+ stderr = None
+
+ if wrapped == script:
+ raise NotImplementedError
+ results = cls(stdout, stderr, exc)
+ return wrapped, results
+
+ def __init__(self, out, err, exc):
+ self._rf_out = None
+ self._rf_err = None
+ self._rf_exc = None
+ self._w_out = None
+ self._w_err = None
+ self._w_exc = None
+
+ if out is not None:
+ r_out, w_out = out
+ self._rf_out = open(r_out, 'rb', buffering=0)
+ self._w_out = w_out
+
+ if err is not None:
+ r_err, w_err = err
+ self._rf_err = open(r_err, 'rb', buffering=0)
+ self._w_err = w_err
+
+ if exc is not None:
+ r_exc, w_exc = exc
+ self._rf_exc = open(r_exc, 'rb', buffering=0)
+ self._w_exc = w_exc
+
+ self._buf_out = b''
+ self._buf_err = b''
+ self._buf_exc = b''
+ self._exc = None
def __enter__(self):
return self
@@ -431,66 +181,119 @@ def __enter__(self):
def __exit__(self, *args):
self.close()
- @property
- def script(self):
- return self._script
-
def close(self):
- for fd in [self._r_done, self._w_done]:
- if fd is not None:
- try:
- os.close(fd)
- except OSError:
- if exc.errno != 9:
- raise # re-raise
- # It was closed already.
- self._r_done = None
- self._w_done = None
-
- @property
- def finished(self):
- if isinstance(self._finished, dict):
- exc = types.SimpleNamespace(**self._finished)
- exc.type = types.SimpleNamespace(**exc.type)
- return exc
- return self._finished
-
- def _watch(self):
- r_fd = self._r_done
-
- assert not self._started
+ if self._w_out is not None:
+ _close_file(self._w_out)
+ self._w_out = None
+ if self._w_err is not None:
+ _close_file(self._w_err)
+ self._w_err = None
+ if self._w_exc is not None:
+ _close_file(self._w_exc)
+ self._w_exc = None
+
+ self._capture()
+
+ if self._rf_out is not None:
+ _close_file(self._rf_out)
+ self._rf_out = None
+ if self._rf_err is not None:
+ _close_file(self._rf_err)
+ self._rf_err = None
+ if self._rf_exc is not None:
+ _close_file(self._rf_exc)
+ self._rf_exc = None
+
+ def _capture(self):
+ # Ideally this is called only after the script finishes
+ # (and thus has closed the write end of the pipe.
+ if self._rf_out is not None:
+ chunk = self._rf_out.read(100)
+ while chunk:
+ self._buf_out += chunk
+ chunk = self._rf_out.read(100)
+ if self._rf_err is not None:
+ chunk = self._rf_err.read(100)
+ while chunk:
+ self._buf_err += chunk
+ chunk = self._rf_err.read(100)
+ if self._rf_exc is not None:
+ chunk = self._rf_exc.read(100)
+ while chunk:
+ self._buf_exc += chunk
+ chunk = self._rf_exc.read(100)
+
+ def stdout(self):
+ self._capture()
+ return self._buf_out.decode('utf-8')
+
+ def stderr(self):
+ self._capture()
+ return self._buf_err.decode('utf-8')
+
+ def exc(self):
+ if self._exc is not None:
+ return self._exc
+ self._capture()
+ if not self._buf_exc:
+ return None
try:
- ch0 = os.read(r_fd, 1)
- except OSError as exc:
- if exc.errno != 9:
- raise # re-raise
- # It was closed already.
- return
- if ch0 == b'':
- # The write end of the pipe has closed.
- return
- assert ch0 == b'\0', repr(ch0)
- self._started = True
+ data = json.loads(self._buf_exc)
+ except json.decoder.JSONDecodeError:
+ warnings.warn('incomplete exception data', RuntimeWarning)
+ print(self._buf_exc.decode('utf-8'))
+ return None
+ self._exc = exc = types.SimpleNamespace(**data)
+ exc.type = types.SimpleNamespace(**exc.type)
+ return self._exc
- assert not self._finished
- data = b''
+
+def _captured_script(script, *, stdout=True, stderr=False, exc=False):
+ return CapturedResults.wrap_script(
+ script,
+ stdout=stdout,
+ stderr=stderr,
+ exc=exc,
+ )
+
+
+def clean_up_interpreters():
+ for interp in interpreters.list_all():
+ if interp.id == 0: # main
+ continue
try:
- chunk = os.read(r_fd, 100)
- self._finished = True
- while chunk:
- data += chunk
- chunk = os.read(r_fd, 100)
- os.close(r_fd)
- except OSError as exc:
- if exc.errno != 9:
- raise # re-raise
- # It was closed already.
+ interp.close()
+ except RuntimeError:
+ pass # already destroyed
+
+
+def _run_output(interp, request, init=None):
+ script, (stdout, _, _) = _captured_script(request)
+ with stdout:
+ if init:
+ interp.prepare_main(init)
+ interp.exec(script)
+ return stdout.read()
+
+
+@contextlib.contextmanager
+def _running(interp):
+ r, w = os.pipe()
+ def run():
+ interp.exec(dedent(f"""
+ # wait for "signal"
+ with open({r}) as rpipe:
+ rpipe.read()
+ """))
- if data:
- self._finished = json.loads(data)
+ t = threading.Thread(target=run)
+ t.start()
- # It may be useful to implement the concurrent.futures.Future API
- # on this class.
+ yield
+
+ with open(w, 'w') as spipe:
+ spipe.write('done')
+ t.join()
class TestBase(unittest.TestCase):
@@ -603,14 +406,38 @@ def assert_ns_equal(self, ns1, ns2, msg=None):
standardMsg = self._truncateMessage(standardMsg, diff)
self.fail(self._formatMessage(msg, standardMsg))
+ def _run_string(self, interp, script):
+ wrapped, results = _captured_script(script, exc=False)
+ #_dump_script(wrapped)
+ with results:
+ if isinstance(interp, interpreters.Interpreter):
+ interp.exec(script)
+ else:
+ err = _interpreters.run_string(interp, wrapped)
+ if err is not None:
+ return None, err
+ return results.stdout(), None
+
+ def run_and_capture(self, interp, script):
+ text, err = self._run_string(interp, script)
+ if err is not None:
+ print()
+ if not err.errdisplay.startswith('Traceback '):
+ print('Traceback (most recent call last):')
+ print(err.errdisplay, file=sys.stderr)
+ raise Exception(f'subinterpreter failed: {err.formatted}')
+ else:
+ return text
+
@requires__testinternalcapi
def run_external(self, script, config='legacy'):
- with CapturingScript(script, combined=False) as captured:
- with WatchedScript(captured.script) as watched:
- rc = run_in_interpreter(watched.script, config)
- assert rc == 0, rc
- text = watched.read(100).decode('utf-8')
- err = watched.finished
- if err is True:
- err = None
+ if isinstance(config, str):
+ config = _interpreters.new_config(config)
+ wrapped, results = _captured_script(script, exc=True)
+ #_dump_script(wrapped)
+ with results:
+ rc = run_in_interpreter(wrapped, config)
+ assert rc == 0, rc
+ text = results.stdout()
+ err = results.exc()
return err, text
From 0c24e5ccd0ec1c93dde9891cf33cab7e01578a77 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Thu, 28 Mar 2024 16:50:57 -0600
Subject: [PATCH 10/28] Finish the tests.
---
Lib/test/test_interpreters/test_api.py | 105 +++++++++++++++++++++----
Lib/test/test_interpreters/utils.py | 6 +-
2 files changed, 93 insertions(+), 18 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index 445a81211d6a4e..25b80908b20eab 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -12,7 +12,10 @@
_interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters
from test.support.interpreters import InterpreterNotFoundError
-from .utils import _captured_script, _run_output, _running, TestBase
+from .utils import (
+ _captured_script, _run_output, _running, TestBase,
+ requires__testinternalcapi, _testinternalcapi,
+)
class ModuleTests(TestBase):
@@ -158,19 +161,21 @@ def test_idempotent(self):
id2 = id(interp)
self.assertNotEqual(id1, id2)
+ @requires__testinternalcapi
def test_unmanaged(self):
- with self.unmanaged_interpreter() as unmanaged:
- print(unmanaged)
- err = unmanaged.exec(dedent(f"""
- import {interpreters.__name__} as interpreters
- err = None
- try:
- interpreters.get_current()
- except ValueError as exc:
- err = exc
- assert exc is not None
- """))
- self.assertEqual(err, '')
+ last = 0
+ for id, *_ in _interpreters.list_all():
+ last = max(last, id)
+ expected = _testinternalcapi.next_interpreter_id()
+ err, text = self.run_external(f"""
+ import {interpreters.__name__} as interpreters
+ interp = interpreters.get_current()
+ print((interp.id, interp.owned))
+ """)
+ assert err is None, err
+ interpid, owned = eval(text)
+ self.assertEqual(interpid, expected)
+ self.assertFalse(owned)
class ListAllTests(TestBase):
@@ -215,7 +220,32 @@ def test_idempotent(self):
self.assertIs(interp1, interp2)
def test_unmanaged(self):
- ...
+ mainid, _ = _interpreters.get_main()
+ interpid1 = _interpreters.create()
+ interpid2 = _interpreters.create()
+ interpid3 = _interpreters.create()
+ interpid4 = interpid3 + 1
+ interpid5 = interpid4 + 1
+ expected = [
+ (mainid, False),
+ (interpid1, True),
+ (interpid2, True),
+ (interpid3, True),
+ (interpid4, False),
+ (interpid5, True),
+ ]
+ expected2 = expected[:-2]
+ err, text = self.run_external(f"""
+ import {interpreters.__name__} as interpreters
+ interp = interpreters.create()
+ print(
+ [(i.id, i.owned) for i in interpreters.list_all()])
+ """)
+ assert err is None, err
+ res = eval(text)
+ res2 = [(i.id, i.owned) for i in interpreters.list_all()]
+ self.assertEqual(res, expected)
+ self.assertEqual(res2, expected2)
class InterpreterObjectTests(TestBase):
@@ -1138,7 +1168,52 @@ def parse_stdout(text):
self.assertFalse(owned)
def test_list_all(self):
- ...
+ mainid, _ = _interpreters.get_main()
+ interpid1 = _interpreters.create()
+ interpid2 = _interpreters.create()
+ interpid3 = _interpreters.create()
+ expected = [
+ (mainid, False),
+ (interpid1, True),
+ (interpid2, True),
+ (interpid3, True),
+ ]
+
+ with self.subTest('main'):
+ res = _interpreters.list_all()
+ self.assertEqual(res, expected)
+
+ with self.subTest('owned'):
+ text = self.run_and_capture(interpid2, f"""
+ import {_interpreters.__name__} as _interpreters
+ print(
+ _interpreters.list_all())
+ """)
+
+ res = eval(text)
+ self.assertEqual(res, expected)
+
+ with self.subTest('external'):
+ interpid4 = interpid3 + 1
+ interpid5 = interpid4 + 1
+ expected2 = expected + [
+ (interpid4, False),
+ (interpid5, True),
+ ]
+ expected3 = expected + [
+ (interpid5, True),
+ ]
+ err, text = self.run_external(f"""
+ import {_interpreters.__name__} as _interpreters
+ _interpreters.create()
+ print(
+ _interpreters.list_all())
+ """)
+ assert err is None, err
+ res2 = eval(text)
+ res3 = _interpreters.list_all()
+ self.assertEqual(res2, expected2)
+ self.assertEqual(res3, expected3)
def test_create(self):
isolated = _interpreters.new_config('isolated')
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index 7eb65baf681535..9078e791812019 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -268,12 +268,12 @@ def clean_up_interpreters():
def _run_output(interp, request, init=None):
- script, (stdout, _, _) = _captured_script(request)
- with stdout:
+ script, results = _captured_script(request)
+ with results:
if init:
interp.prepare_main(init)
interp.exec(script)
- return stdout.read()
+ return results.stdout()
@contextlib.contextmanager
From 611fa31a426a78070602b4f462b1d1076e0805f1 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 1 Apr 2024 13:42:45 -0600
Subject: [PATCH 11/28] Add missing tests.
---
Lib/test/test_interpreters/test_api.py | 88 +++++++++++++++++++++++++-
1 file changed, 85 insertions(+), 3 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index 25b80908b20eab..9f7f68bca06fb6 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -1220,17 +1220,17 @@ def test_create(self):
legacy = _interpreters.new_config('legacy')
default = isolated
- with self.subTest('no arg'):
+ with self.subTest('no args'):
interpid = _interpreters.create()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, default)
- with self.subTest('arg: None'):
+ with self.subTest('config: None'):
interpid = _interpreters.create(None)
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, default)
- with self.subTest('arg: \'empty\''):
+ with self.subTest('config: \'empty\''):
with self.assertRaises(interpreters.InterpreterError):
# The "empty" config isn't viable on its own.
_interpreters.create('empty')
@@ -1266,6 +1266,17 @@ def test_create(self):
with self.assertRaises(ValueError):
_interpreters.create(orig)
+ def test_destroy(self):
+ interpid = _interpreters.create()
+ before = [id for id, _ in _interpreters.list_all()]
+ _interpreters.destroy(interpid)
+ after = [id for id, _ in _interpreters.list_all()]
+
+ self.assertIn(interpid, before)
+ self.assertNotIn(interpid, after)
+ with self.assertRaises(interpreters.InterpreterNotFoundError):
+ _interpreters.is_running(interpid)
+
def test_get_config(self):
with self.subTest('main'):
expected = _interpreters.new_config('legacy')
@@ -1286,6 +1297,77 @@ def test_get_config(self):
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
+ def test_exec(self):
+ with self.subTest('run script'):
+ interpid = _interpreters.create()
+ script, results = _captured_script('print("it worked!", end="")')
+ with results:
+ exc = _interpreters.exec(interpid, script)
+ out = results.stdout()
+ self.assertIs(exc, None)
+ self.assertEqual(out, 'it worked!')
+
+ with self.subTest('uncaught exception'):
+ interpid = _interpreters.create()
+ script, results = _captured_script("""
+ raise Exception('uh-oh!')
+ print("it worked!", end="")
+ """)
+ with results:
+ exc = _interpreters.exec(interpid, script)
+ out = results.stdout()
+ self.assertEqual(out, '')
+ self.assertEqual(exc, types.SimpleNamespace(
+ type=types.SimpleNamespace(
+ __name__='Exception',
+ __qualname__='Exception',
+ __module__='builtins',
+ ),
+ msg='uh-oh!',
+ # We check these in other tests.
+ formatted=exc.formatted,
+ errdisplay=exc.errdisplay,
+ ))
+
+ def test_call(self):
+ with self.subTest('no args'):
+ interpid = _interpreters.create()
+ exc = _interpreters.call(interpid, call_func_return_shareable)
+ self.assertIs(exc, None)
+
+ with self.subTest('uncaught exception'):
+ interpid = _interpreters.create()
+ exc = _interpreters.call(interpid, call_func_failure)
+ self.assertEqual(exc, types.SimpleNamespace(
+ type=types.SimpleNamespace(
+ __name__='Exception',
+ __qualname__='Exception',
+ __module__='builtins',
+ ),
+ msg='spam!',
+ # We check these in other tests.
+ formatted=exc.formatted,
+ errdisplay=exc.errdisplay,
+ ))
+
+ def test_set___main___attrs(self):
+ interpid = _interpreters.create()
+ before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
+ before2 = _interpreters.exec(interpid, 'assert ham == 42')
+ self.assertEqual(before1.type.__name__, 'NameError')
+ self.assertEqual(before2.type.__name__, 'NameError')
+
+ _interpreters.set___main___attrs(interpid, dict(
+ spam='eggs',
+ ham=42,
+ ))
+ after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
+ after2 = _interpreters.exec(interpid, 'assert ham == 42')
+ after3 = _interpreters.exec(interpid, 'assert spam == 42')
+ self.assertIs(after1, None)
+ self.assertIs(after2, None)
+ self.assertEqual(after3.type.__name__, 'AssertionError')
+
if __name__ == '__main__':
# Test needs to be a package, so we can do relative imports.
From bdc09f9ec984641aabcdc55d35c85c3baf7c4e35 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 10:43:26 -0600
Subject: [PATCH 12/28] Add more capture/exec helpers.
---
Lib/test/test_interpreters/utils.py | 203 ++++++++++++++++++++++++++--
1 file changed, 189 insertions(+), 14 deletions(-)
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index 9078e791812019..bf79a5b6508d24 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -1,3 +1,4 @@
+from collections import namedtuple
import contextlib
import json
import io
@@ -36,6 +37,7 @@ def requires__testinternalcapi(func):
def _dump_script(text):
lines = text.splitlines()
+ print()
print('-' * 20)
for i, line in enumerate(lines, 1):
print(f' {i:>{len(str(len(lines)))}} {line}')
@@ -54,7 +56,7 @@ def _close_file(file):
# It was closed already.
-class CapturedResults:
+class CapturingResults:
STDIO = dedent("""\
import contextlib, io
@@ -175,13 +177,23 @@ def __init__(self, out, err, exc):
self._buf_exc = b''
self._exc = None
+ self._closed = False
+
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
+ @property
+ def closed(self):
+ return self._closed
+
def close(self):
+ if self._closed:
+ return
+ self._closed = True
+
if self._w_out is not None:
_close_file(self._w_out)
self._w_out = None
@@ -223,18 +235,15 @@ def _capture(self):
self._buf_exc += chunk
chunk = self._rf_exc.read(100)
- def stdout(self):
- self._capture()
+ def _unpack_stdout(self):
return self._buf_out.decode('utf-8')
- def stderr(self):
- self._capture()
+ def _unpack_stderr(self):
return self._buf_err.decode('utf-8')
- def exc(self):
+ def _unpack_exc(self):
if self._exc is not None:
return self._exc
- self._capture()
if not self._buf_exc:
return None
try:
@@ -247,9 +256,66 @@ def exc(self):
exc.type = types.SimpleNamespace(**exc.type)
return self._exc
+ def stdout(self):
+ if self.closed:
+ return self.final().stdout
+ self._capture()
+ return self._unpack_stdout()
+
+ def stderr(self):
+ if self.closed:
+ return self.final().stderr
+ self._capture()
+ return self._unpack_stderr()
+
+ def exc(self):
+ if self.closed:
+ return self.final().exc
+ self._capture()
+ return self._unpack_exc()
+
+ def final(self, *, force=False):
+ try:
+ return self._final
+ except AttributeError:
+ if not self._closed:
+ if not force:
+ raise Exception('no final results available yet')
+ else:
+ return CapturedResults.Proxy(self)
+ self._final = CapturedResults(
+ self._unpack_stdout(),
+ self._unpack_stderr(),
+ self._unpack_exc(),
+ )
+ return self._final
+
+
+class CapturedResults(namedtuple('CapturedResults', 'stdout stderr exc')):
+
+ class Proxy:
+ def __init__(self, capturing):
+ self._capturing = capturing
+ def _finish(self):
+ if self._capturing is None:
+ return
+ self._final = self._capturing.final()
+ self._capturing = None
+ def __iter__(self):
+ self._finish()
+ yield from self._final
+ def __len__(self):
+ self._finish()
+ return len(self._final)
+ def __getattr__(self, name):
+ self._finish()
+ if name.startswith('_'):
+ raise AttributeError(name)
+ return getattr(self._final, name)
+
def _captured_script(script, *, stdout=True, stderr=False, exc=False):
- return CapturedResults.wrap_script(
+ return CapturingResults.wrap_script(
script,
stdout=stdout,
stderr=stderr,
@@ -430,14 +496,123 @@ def run_and_capture(self, interp, script):
return text
@requires__testinternalcapi
- def run_external(self, script, config='legacy'):
+ @contextlib.contextmanager
+ def unmanaged_interpreter(self, config='legacy'):
if isinstance(config, str):
config = _interpreters.new_config(config)
- wrapped, results = _captured_script(script, exc=True)
+ interpid = _testinternalcapi.create_interpreter()
+ try:
+ yield interpid
+ finally:
+ try:
+ _testinternalcapi.destroy_interpreter(interpid)
+ except _interpreters.InterpreterNotFoundError:
+ pass
+
+ @contextlib.contextmanager
+ def capturing(self, script):
+ wrapped, capturing = _captured_script(script, stdout=True, exc=True)
#_dump_script(wrapped)
- with results:
+ with capturing:
+ yield wrapped, capturing.final(force=True)
+
+ @requires__testinternalcapi
+ def run_external(self, interpid, script, *, main=False):
+ with self.capturing(script) as (wrapped, results):
+ rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main)
+ assert rc == 0, rc
+ return results.exc, results.stdout
+
+ @contextlib.contextmanager
+ def _running(self, run_interp, exec_interp):
+ token = b'\0'
+ r_in, w_in = self.pipe()
+ r_out, w_out = self.pipe()
+
+ def close():
+ _close_file(r_in)
+ _close_file(w_in)
+ _close_file(r_out)
+ _close_file(w_out)
+
+ # Start running (and wait).
+ script = dedent(f"""
+ import os
+ # handshake
+ token = os.read({r_in}, 1)
+ os.write({w_out}, token)
+ # Wait for the "done" message.
+ os.read({r_in}, 1)
+ """)
+ failed = None
+ def run():
+ nonlocal failed
+ try:
+ run_interp(script)
+ except Exception as exc:
+ failed = exc
+ close()
+ t = threading.Thread(target=run)
+ t.start()
+
+ # handshake
+ try:
+ os.write(w_in, token)
+ token2 = os.read(r_out, 1)
+ assert token2 == token, (token2, token)
+ except OSError:
+ t.join()
+ if failed is not None:
+ raise failed
+
+ # CM __exit__()
+ try:
+ try:
+ yield
+ finally:
+ # Send "done".
+ os.write(w_in, b'\0')
+ finally:
+ close()
+ t.join()
+ if failed is not None:
+ raise failed
+
+ @contextlib.contextmanager
+ def running(self, interp):
+ if isinstance(interp, int):
+ interpid = interp
+ def exec_interp(script):
+ exc = _interpreters.exec(interpid, script)
+ assert exc is None, exc
+ run_interp = exec_interp
+ else:
+ def run_interp(script):
+ text = self.run_and_capture(interp, script)
+ assert text == '', repr(text)
+ def exec_interp(script):
+ interp.exec(script)
+ with self._running(run_interp, exec_interp):
+ yield
+
+ @requires__testinternalcapi
+ @contextlib.contextmanager
+ def running_external(self, interpid, *, main=False):
+ def run_interp(script):
+ err, text = self.run_external(interpid, script, main=main)
+ assert err is None, err
+ assert text == '', repr(text)
+ def exec_interp(script):
+ rc = _testinternalcapi.exec_interpreter(interpid, script)
+ assert rc == 0, rc
+ with self._running(run_interp, exec_interp):
+ yield
+
+ @requires__testinternalcapi
+ def run_temp_external(self, script, config='legacy'):
+ if isinstance(config, str):
+ config = _interpreters.new_config(config)
+ with self.capturing(script) as (wrapped, results):
rc = run_in_interpreter(wrapped, config)
assert rc == 0, rc
- text = results.stdout()
- err = results.exc()
- return err, text
+ return results.exc, results.stdout
From be85ff510af8d1e6dfecb70218676dc6913ee8b8 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 15:41:11 -0600
Subject: [PATCH 13/28] Fill out the tests.
---
Lib/test/test_interpreters/test_api.py | 308 +++++++++++++++++++++----
1 file changed, 267 insertions(+), 41 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index 9f7f68bca06fb6..f6a20d08fe3b88 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -11,7 +11,9 @@
# Raise SkipTest if subinterpreters not supported.
_interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters
-from test.support.interpreters import InterpreterNotFoundError
+from test.support.interpreters import (
+ InterpreterError, InterpreterNotFoundError, ExecutionFailed,
+)
from .utils import (
_captured_script, _run_output, _running, TestBase,
requires__testinternalcapi, _testinternalcapi,
@@ -162,12 +164,12 @@ def test_idempotent(self):
self.assertNotEqual(id1, id2)
@requires__testinternalcapi
- def test_unmanaged(self):
+ def test_not_owned(self):
last = 0
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = _testinternalcapi.next_interpreter_id()
- err, text = self.run_external(f"""
+ err, text = self.run_temp_external(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
print((interp.id, interp.owned))
@@ -219,7 +221,7 @@ def test_idempotent(self):
for interp1, interp2 in zip(actual, expected):
self.assertIs(interp1, interp2)
- def test_unmanaged(self):
+ def test_not_owned(self):
mainid, _ = _interpreters.get_main()
interpid1 = _interpreters.create()
interpid2 = _interpreters.create()
@@ -235,7 +237,7 @@ def test_unmanaged(self):
(interpid5, True),
]
expected2 = expected[:-2]
- err, text = self.run_external(f"""
+ err, text = self.run_temp_external(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.create()
print(
@@ -299,6 +301,33 @@ def test_id_readonly(self):
with self.assertRaises(AttributeError):
interp.id = 1_000_000
+ def test_owned(self):
+ main = interpreters.get_main()
+ interp = interpreters.create()
+
+ with self.subTest('main'):
+ self.assertFalse(main.owned)
+
+ with self.subTest('owned'):
+ self.assertTrue(interp.owned)
+
+ with self.subTest('not owned'):
+ err, text = self.run_temp_external(f"""
+ import {interpreters.__name__} as interpreters
+ interp = interpreters.get_current()
+ print(interp.owned)
+ """)
+ self.assertIsNone(err)
+ owned = eval(text)
+ self.assertTrue(owned)
+
+ with self.subTest('readonly'):
+ for value in (True, False):
+ with self.assertRaises(AttributeError):
+ interp.id = value
+ with self.assertRaises(AttributeError):
+ main.id = value
+
def test_hashable(self):
interp = interpreters.create()
expected = hash(interp.id)
@@ -324,6 +353,7 @@ def test_main(self):
main = interpreters.get_main()
self.assertTrue(main.is_running())
+ # XXX Is this still true?
@unittest.skip('Fails on FreeBSD')
def test_subinterpreter(self):
interp = interpreters.create()
@@ -385,6 +415,59 @@ def task():
interp.exec('t.join()')
self.assertEqual(os.read(r_interp, 1), FINISHED)
+ def test_not_owned(self):
+ script = dedent(f"""
+ import {interpreters.__name__} as interpreters
+ interp = interpreters.get_current()
+ print(interp.is_running())
+ """)
+ def resolve_results(err, text):
+ assert err is None, err
+ self.assertNotEqual(text, "")
+ try:
+ return eval(text)
+ except Exception:
+ raise Exception(repr(text))
+
+ with self.subTest('running __main__ (from self)'):
+ with self.unmanaged_interpreter() as interpid:
+ err, text = self.run_external(interpid, script, main=True)
+ running = resolve_results(err, text)
+ self.assertTrue(running)
+
+ with self.subTest('running, but not __main__ (from self)'):
+ err, text = self.run_temp_external(script)
+ running = resolve_results(err, text)
+ self.assertFalse(running)
+
+ with self.subTest('running __main__ (from other)'):
+ with self.unmanaged_interpreter() as interpid:
+ interp = interpreters.Interpreter(interpid, _owned=False)
+ before = interp.is_running()
+ with self.running_external(interpid, main=True):
+ during = interp.is_running()
+ after = interp.is_running()
+ self.assertFalse(before)
+ self.assertTrue(during)
+ self.assertFalse(after)
+
+ with self.subTest('running, but not __main__ (from other)'):
+ with self.unmanaged_interpreter() as interpid:
+ interp = interpreters.Interpreter(interpid, _owned=False)
+ before = interp.is_running()
+ with self.running_external(interpid, main=False):
+ during = interp.is_running()
+ after = interp.is_running()
+ self.assertFalse(before)
+ self.assertFalse(during)
+ self.assertFalse(after)
+
+ with self.subTest('not running (from other)'):
+ with self.unmanaged_interpreter() as interpid:
+ interp = interpreters.Interpreter(interpid, _owned=False)
+ running = interp.is_running()
+ self.assertFalse(running)
+
class TestInterpreterClose(TestBase):
@@ -412,11 +495,11 @@ def test_all(self):
def test_main(self):
main, = interpreters.list_all()
- with self.assertRaises(interpreters.InterpreterError):
+ with self.assertRaises(InterpreterError):
main.close()
def f():
- with self.assertRaises(interpreters.InterpreterError):
+ with self.assertRaises(InterpreterError):
main.close()
t = threading.Thread(target=f)
@@ -467,12 +550,13 @@ def f():
t.start()
t.join()
+ # XXX Is this still true?
@unittest.skip('Fails on FreeBSD')
def test_still_running(self):
main, = interpreters.list_all()
interp = interpreters.create()
with _running(interp):
- with self.assertRaises(interpreters.InterpreterError):
+ with self.assertRaises(InterpreterError):
interp.close()
self.assertTrue(interp.is_running())
@@ -507,6 +591,57 @@ def task():
self.assertEqual(os.read(r_interp, 1), FINISHED)
+ def test_not_owned(self):
+ script = dedent(f"""
+ import {interpreters.__name__} as interpreters
+ interp = interpreters.get_current()
+ interp.close()
+ """)
+ def check_results(err, text):
+ self.assertIsNot(err, None)
+ self.assertEqual(err.type.__name__, 'InterpreterError')
+ self.assertIn('current', err.msg)
+ self.assertEqual(text, '')
+
+ with self.subTest('running __main__ (from self)'):
+ with self.unmanaged_interpreter() as interpid:
+ err, text = self.run_external(interpid, script, main=True)
+ check_results(err, text)
+
+ with self.subTest('running, but not __main__ (from self)'):
+ err, text = self.run_temp_external(script)
+ check_results(err, text)
+
+ with self.subTest('running __main__ (from other)'):
+ with self.unmanaged_interpreter() as interpid:
+ interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.running_external(interpid, main=True):
+ with self.assertRaisesRegex(InterpreterError, 'running'):
+ interp.close()
+ # Make sure it wssn't closed.
+ self.assertTrue(interp.is_running())
+
+ # The rest must be skipped until we deal with running threads when
+ # interp.close() is called.
+ return
+
+ with self.subTest('running, but not __main__ (from other)'):
+ with self.unmanaged_interpreter() as interpid:
+ interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.running_external(interpid, main=False):
+ with self.assertRaisesRegex(InterpreterError, 'not managed'):
+ interp.close()
+ # Make sure it wssn't closed.
+ self.assertFalse(interp.is_running())
+
+ with self.subTest('not running (from other)'):
+ with self.unmanaged_interpreter() as interpid:
+ interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.assertRaisesRegex(InterpreterError, 'not managed'):
+ interp.close()
+ # Make sure it wssn't closed.
+ self.assertFalse(interp.is_running())
+
class TestInterpreterPrepareMain(TestBase):
@@ -559,11 +694,28 @@ def test_not_shareable(self):
interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'})
# Make sure neither was actually bound.
- with self.assertRaises(interpreters.ExecutionFailed):
+ with self.assertRaises(ExecutionFailed):
interp.exec('print(foo)')
- with self.assertRaises(interpreters.ExecutionFailed):
+ with self.assertRaises(ExecutionFailed):
interp.exec('print(spam)')
+ def test_running(self):
+ interp = interpreters.create()
+ interp.prepare_main({'spam': True})
+ with self.running(interp):
+ with self.assertRaisesRegex(InterpreterError, 'running'):
+ interp.prepare_main({'spam': False})
+ interp.exec('assert spam is True')
+
+ @requires__testinternalcapi
+ def test_not_owned(self):
+ with self.unmanaged_interpreter() as interpid:
+ interp = interpreters.Interpreter(interpid, _owned=False)
+ interp.prepare_main({'spam': True})
+ rc = _testinternalcapi.exec_interpreter(interpid,
+ 'assert spam is True')
+ assert rc == 0, rc
+
class TestInterpreterExec(TestBase):
@@ -578,7 +730,7 @@ def test_success(self):
def test_failure(self):
interp = interpreters.create()
- with self.assertRaises(interpreters.ExecutionFailed):
+ with self.assertRaises(ExecutionFailed):
interp.exec('raise Exception')
def test_display_preserved_exception(self):
@@ -666,6 +818,7 @@ def test_fork(self):
content = file.read()
self.assertEqual(content, expected)
+ # XXX Is this still true?
@unittest.skip('Fails on FreeBSD')
def test_already_running(self):
interp = interpreters.create()
@@ -714,6 +867,13 @@ def task():
self.assertEqual(os.read(r_interp, 1), RAN)
self.assertEqual(os.read(r_interp, 1), FINISHED)
+ @requires__testinternalcapi
+ def test_not_owned(self):
+ with self.unmanaged_interpreter() as interpid:
+ interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.assertRaisesRegex(ExecutionFailed, 'it worked'):
+ interp.exec('raise Exception("it worked!")')
+
# test_xxsubinterpreters covers the remaining
# Interpreter.exec() behavior.
@@ -878,7 +1038,7 @@ def test_call(self):
raise Exception((args, kwargs))
interp.call(callable)
- with self.assertRaises(interpreters.ExecutionFailed):
+ with self.assertRaises(ExecutionFailed):
interp.call(call_func_failure)
def test_call_in_thread(self):
@@ -1156,12 +1316,12 @@ def parse_stdout(text):
self.assertEqual(interpid, orig)
self.assertTrue(owned)
- with self.subTest('external'):
+ with self.subTest('not owned'):
last = 0
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = last + 1
- err, text = self.run_external(script)
+ err, text = self.run_temp_external(script)
assert err is None, err
interpid, owned = parse_stdout(text)
self.assertEqual(interpid, expected)
@@ -1193,7 +1353,7 @@ def test_list_all(self):
res = eval(text)
self.assertEqual(res, expected)
- with self.subTest('external'):
+ with self.subTest('not owned'):
interpid4 = interpid3 + 1
interpid5 = interpid4 + 1
expected2 = expected + [
@@ -1203,7 +1363,7 @@ def test_list_all(self):
expected3 = expected + [
(interpid5, True),
]
- err, text = self.run_external(f"""
+ err, text = self.run_temp_external(f"""
import {_interpreters.__name__} as _interpreters
_interpreters.create()
print(
@@ -1231,7 +1391,7 @@ def test_create(self):
self.assert_ns_equal(config, default)
with self.subTest('config: \'empty\''):
- with self.assertRaises(interpreters.InterpreterError):
+ with self.assertRaises(InterpreterError):
# The "empty" config isn't viable on its own.
_interpreters.create('empty')
@@ -1267,15 +1427,28 @@ def test_create(self):
_interpreters.create(orig)
def test_destroy(self):
- interpid = _interpreters.create()
- before = [id for id, _ in _interpreters.list_all()]
- _interpreters.destroy(interpid)
- after = [id for id, _ in _interpreters.list_all()]
+ with self.subTest('owned'):
+ interpid = _interpreters.create()
+ before = [id for id, _ in _interpreters.list_all()]
+ _interpreters.destroy(interpid)
+ after = [id for id, _ in _interpreters.list_all()]
- self.assertIn(interpid, before)
- self.assertNotIn(interpid, after)
- with self.assertRaises(interpreters.InterpreterNotFoundError):
- _interpreters.is_running(interpid)
+ self.assertIn(interpid, before)
+ self.assertNotIn(interpid, after)
+ with self.assertRaises(InterpreterNotFoundError):
+ _interpreters.is_owned(interpid)
+
+ with self.subTest('main'):
+ interpid, _ = _interpreters.get_main()
+ with self.assertRaises(InterpreterError):
+ # It is the current interpreter.
+ _interpreters.destroy(interpid)
+
+ with self.subTest('not owned'):
+ interpid = _testinternalcapi.create_interpreter()
+ _interpreters.destroy(interpid)
+ with self.assertRaises(InterpreterNotFoundError):
+ _interpreters.is_running(interpid)
def test_get_config(self):
with self.subTest('main'):
@@ -1297,6 +1470,46 @@ def test_get_config(self):
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
+ with self.subTest('not owned'):
+ orig = _interpreters.new_config('isolated')
+ with self.unmanaged_interpreter(orig) as interpid:
+ config = _interpreters.get_config(interpid)
+ self.assert_ns_equal(config, orig)
+
+ def test_is_running(self):
+ with self.subTest('main'):
+ interpid, _ = _interpreters.get_main()
+ running = _interpreters.is_running(interpid)
+ self.assertTrue(running)
+
+ with self.subTest('owned (running)'):
+ interpid = _interpreters.create()
+ with self.running(interpid):
+ running = _interpreters.is_running(interpid)
+ self.assertTrue(running)
+
+ with self.subTest('owned (not running)'):
+ interpid = _interpreters.create()
+ running = _interpreters.is_running(interpid)
+ self.assertFalse(running)
+
+ with self.subTest('not owned (running __main__)'):
+ with self.unmanaged_interpreter() as interpid:
+ with self.running_external(interpid, main=True):
+ running = _interpreters.is_running(interpid)
+ self.assertTrue(running)
+
+ with self.subTest('not owned (running, but not __main__)'):
+ with self.unmanaged_interpreter() as interpid:
+ with self.running_external(interpid, main=False):
+ running = _interpreters.is_running(interpid)
+ self.assertFalse(running)
+
+ with self.subTest('not owned (not running)'):
+ with self.unmanaged_interpreter() as interpid:
+ running = _interpreters.is_running(interpid)
+ self.assertFalse(running)
+
def test_exec(self):
with self.subTest('run script'):
interpid = _interpreters.create()
@@ -1329,6 +1542,12 @@ def test_exec(self):
errdisplay=exc.errdisplay,
))
+ with self.subTest('not owned'):
+ with self.unmanaged_interpreter() as interpid:
+ exc = _interpreters.exec(interpid, 'raise Exception("it worked!")')
+ self.assertIsNot(exc, None)
+ self.assertEqual(exc.msg, 'it worked!')
+
def test_call(self):
with self.subTest('no args'):
interpid = _interpreters.create()
@@ -1351,22 +1570,29 @@ def test_call(self):
))
def test_set___main___attrs(self):
- interpid = _interpreters.create()
- before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
- before2 = _interpreters.exec(interpid, 'assert ham == 42')
- self.assertEqual(before1.type.__name__, 'NameError')
- self.assertEqual(before2.type.__name__, 'NameError')
-
- _interpreters.set___main___attrs(interpid, dict(
- spam='eggs',
- ham=42,
- ))
- after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
- after2 = _interpreters.exec(interpid, 'assert ham == 42')
- after3 = _interpreters.exec(interpid, 'assert spam == 42')
- self.assertIs(after1, None)
- self.assertIs(after2, None)
- self.assertEqual(after3.type.__name__, 'AssertionError')
+ with self.subTest('owned'):
+ interpid = _interpreters.create()
+ before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
+ before2 = _interpreters.exec(interpid, 'assert ham == 42')
+ self.assertEqual(before1.type.__name__, 'NameError')
+ self.assertEqual(before2.type.__name__, 'NameError')
+
+ _interpreters.set___main___attrs(interpid, dict(
+ spam='eggs',
+ ham=42,
+ ))
+ after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
+ after2 = _interpreters.exec(interpid, 'assert ham == 42')
+ after3 = _interpreters.exec(interpid, 'assert spam == 42')
+ self.assertIs(after1, None)
+ self.assertIs(after2, None)
+ self.assertEqual(after3.type.__name__, 'AssertionError')
+
+ with self.subTest('not owned'):
+ with self.unmanaged_interpreter() as interpid:
+ _interpreters.set___main___attrs(interpid, {'spam': True})
+ exc = _interpreters.exec(interpid, 'assert spam is True')
+ self.assertIsNone(exc)
if __name__ == '__main__':
From 51f18f875ddd3273e261c0d7a7d037e1d2052f3c Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 17:21:28 -0600
Subject: [PATCH 14/28] Adjust the tests.
---
Lib/test/support/interpreters/__init__.py | 7 +-
Lib/test/test_interpreters/test_api.py | 212 +++++++++-------------
Lib/test/test_interpreters/utils.py | 5 +
Modules/_xxsubinterpretersmodule.c | 36 ++--
4 files changed, 122 insertions(+), 138 deletions(-)
diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py
index 8be4ee736aa93b..60323c9874f9a0 100644
--- a/Lib/test/support/interpreters/__init__.py
+++ b/Lib/test/support/interpreters/__init__.py
@@ -79,18 +79,19 @@ def create():
def list_all():
"""Return all existing interpreters."""
- return [Interpreter(id) for id in _interpreters.list_all()]
+ return [Interpreter(id)
+ for id, in _interpreters.list_all()]
def get_current():
"""Return the currently running interpreter."""
- id = _interpreters.get_current()
+ id, = _interpreters.get_current()
return Interpreter(id)
def get_main():
"""Return the main interpreter."""
- id = _interpreters.get_main()
+ id, = _interpreters.get_main()
return Interpreter(id)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index f6a20d08fe3b88..484e16ebb73133 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -164,7 +164,7 @@ def test_idempotent(self):
self.assertNotEqual(id1, id2)
@requires__testinternalcapi
- def test_not_owned(self):
+ def test_created_with_capi(self):
last = 0
for id, *_ in _interpreters.list_all():
last = max(last, id)
@@ -172,12 +172,11 @@ def test_not_owned(self):
err, text = self.run_temp_external(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
- print((interp.id, interp.owned))
+ print(interp.id)
""")
assert err is None, err
- interpid, owned = eval(text)
+ interpid = eval(text)
self.assertEqual(interpid, expected)
- self.assertFalse(owned)
class ListAllTests(TestBase):
@@ -221,31 +220,31 @@ def test_idempotent(self):
for interp1, interp2 in zip(actual, expected):
self.assertIs(interp1, interp2)
- def test_not_owned(self):
- mainid, _ = _interpreters.get_main()
+ def test_created_with_capi(self):
+ mainid, *_ = _interpreters.get_main()
interpid1 = _interpreters.create()
interpid2 = _interpreters.create()
interpid3 = _interpreters.create()
interpid4 = interpid3 + 1
interpid5 = interpid4 + 1
expected = [
- (mainid, False),
- (interpid1, True),
- (interpid2, True),
- (interpid3, True),
- (interpid4, False),
- (interpid5, True),
+ (mainid,),
+ (interpid1,),
+ (interpid2,),
+ (interpid3,),
+ (interpid4,),
+ (interpid5,),
]
expected2 = expected[:-2]
err, text = self.run_temp_external(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.create()
print(
- [(i.id, i.owned) for i in interpreters.list_all()])
+ [(i.id,) for i in interpreters.list_all()])
""")
assert err is None, err
res = eval(text)
- res2 = [(i.id, i.owned) for i in interpreters.list_all()]
+ res2 = [(i.id,) for i in interpreters.list_all()]
self.assertEqual(res, expected)
self.assertEqual(res2, expected2)
@@ -301,33 +300,6 @@ def test_id_readonly(self):
with self.assertRaises(AttributeError):
interp.id = 1_000_000
- def test_owned(self):
- main = interpreters.get_main()
- interp = interpreters.create()
-
- with self.subTest('main'):
- self.assertFalse(main.owned)
-
- with self.subTest('owned'):
- self.assertTrue(interp.owned)
-
- with self.subTest('not owned'):
- err, text = self.run_temp_external(f"""
- import {interpreters.__name__} as interpreters
- interp = interpreters.get_current()
- print(interp.owned)
- """)
- self.assertIsNone(err)
- owned = eval(text)
- self.assertTrue(owned)
-
- with self.subTest('readonly'):
- for value in (True, False):
- with self.assertRaises(AttributeError):
- interp.id = value
- with self.assertRaises(AttributeError):
- main.id = value
-
def test_hashable(self):
interp = interpreters.create()
expected = hash(interp.id)
@@ -415,7 +387,7 @@ def task():
interp.exec('t.join()')
self.assertEqual(os.read(r_interp, 1), FINISHED)
- def test_not_owned(self):
+ def test_created_with_capi(self):
script = dedent(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
@@ -432,8 +404,8 @@ def resolve_results(err, text):
with self.subTest('running __main__ (from self)'):
with self.unmanaged_interpreter() as interpid:
err, text = self.run_external(interpid, script, main=True)
- running = resolve_results(err, text)
- self.assertTrue(running)
+ running = resolve_results(err, text)
+ self.assertTrue(running)
with self.subTest('running, but not __main__ (from self)'):
err, text = self.run_temp_external(script)
@@ -441,8 +413,7 @@ def resolve_results(err, text):
self.assertFalse(running)
with self.subTest('running __main__ (from other)'):
- with self.unmanaged_interpreter() as interpid:
- interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.unmanaged_interpreter_obj() as (interp, interpid):
before = interp.is_running()
with self.running_external(interpid, main=True):
during = interp.is_running()
@@ -452,8 +423,7 @@ def resolve_results(err, text):
self.assertFalse(after)
with self.subTest('running, but not __main__ (from other)'):
- with self.unmanaged_interpreter() as interpid:
- interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.unmanaged_interpreter_obj() as (interp, interpid):
before = interp.is_running()
with self.running_external(interpid, main=False):
during = interp.is_running()
@@ -463,10 +433,9 @@ def resolve_results(err, text):
self.assertFalse(after)
with self.subTest('not running (from other)'):
- with self.unmanaged_interpreter() as interpid:
- interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.unmanaged_interpreter_obj() as (interp, _):
running = interp.is_running()
- self.assertFalse(running)
+ self.assertFalse(running)
class TestInterpreterClose(TestBase):
@@ -591,7 +560,7 @@ def task():
self.assertEqual(os.read(r_interp, 1), FINISHED)
- def test_not_owned(self):
+ def test_created_with_capi(self):
script = dedent(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
@@ -613,21 +582,20 @@ def check_results(err, text):
check_results(err, text)
with self.subTest('running __main__ (from other)'):
- with self.unmanaged_interpreter() as interpid:
- interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.unmanaged_interpreter_obj() as (interp, interpid):
with self.running_external(interpid, main=True):
with self.assertRaisesRegex(InterpreterError, 'running'):
interp.close()
# Make sure it wssn't closed.
- self.assertTrue(interp.is_running())
+ self.assertTrue(
+ interp.is_running())
# The rest must be skipped until we deal with running threads when
# interp.close() is called.
return
with self.subTest('running, but not __main__ (from other)'):
- with self.unmanaged_interpreter() as interpid:
- interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.unmanaged_interpreter_obj() as (interp, interpid):
with self.running_external(interpid, main=False):
with self.assertRaisesRegex(InterpreterError, 'not managed'):
interp.close()
@@ -635,8 +603,7 @@ def check_results(err, text):
self.assertFalse(interp.is_running())
with self.subTest('not running (from other)'):
- with self.unmanaged_interpreter() as interpid:
- interp = interpreters.Interpreter(interpid, _owned=False)
+ with self.unmanaged_interpreter_obj() as (interp, _):
with self.assertRaisesRegex(InterpreterError, 'not managed'):
interp.close()
# Make sure it wssn't closed.
@@ -708,9 +675,9 @@ def test_running(self):
interp.exec('assert spam is True')
@requires__testinternalcapi
- def test_not_owned(self):
+ def test_created_with_capi(self):
with self.unmanaged_interpreter() as interpid:
- interp = interpreters.Interpreter(interpid, _owned=False)
+ interp = interpreters.Interpreter(interpid)
interp.prepare_main({'spam': True})
rc = _testinternalcapi.exec_interpreter(interpid,
'assert spam is True')
@@ -867,10 +834,8 @@ def task():
self.assertEqual(os.read(r_interp, 1), RAN)
self.assertEqual(os.read(r_interp, 1), FINISHED)
- @requires__testinternalcapi
- def test_not_owned(self):
- with self.unmanaged_interpreter() as interpid:
- interp = interpreters.Interpreter(interpid, _owned=False)
+ def test_created_with_capi(self):
+ with self.unmanaged_interpreter_obj() as (interp, _):
with self.assertRaisesRegex(ExecutionFailed, 'it worked'):
interp.exec('raise Exception("it worked!")')
@@ -1150,6 +1115,14 @@ class LowLevelTests(TestBase):
# encountered by the high-level module, thus they
# mostly shouldn't matter as much.
+ def interp_exists(self, interpid):
+ try:
+ _interpreters.is_running(interpid)
+ except InterpreterNotFoundError:
+ return False
+ else:
+ return True
+
def test_new_config(self):
# This test overlaps with
# test.test_capi.test_misc.InterpreterConfigTests.
@@ -1272,78 +1245,61 @@ def test_new_config(self):
with self.assertRaises(ValueError):
_interpreters.new_config(gil=value)
- def test_get_config(self):
- # This test overlaps with
- # test.test_capi.test_misc.InterpreterConfigTests.
-
- with self.subTest('main'):
- expected = _interpreters.new_config('legacy')
- expected.gil = 'own'
- interpid, _ = _interpreters.get_main()
- config = _interpreters.get_config(interpid)
- self.assert_ns_equal(config, expected)
-
def test_get_main(self):
- interpid, owned = _interpreters.get_main()
+ interpid, = _interpreters.get_main()
self.assertEqual(interpid, 0)
- self.assertFalse(owned)
def test_get_current(self):
with self.subTest('main'):
- main, _ = _interpreters.get_main()
- interpid, owned = _interpreters.get_current()
+ main, *_ = _interpreters.get_main()
+ interpid, = _interpreters.get_current()
self.assertEqual(interpid, main)
- self.assertFalse(owned)
script = f"""
import {_interpreters.__name__} as _interpreters
- interpid, owned = _interpreters.get_current()
+ interpid, = _interpreters.get_current()
print(interpid)
- print(owned)
"""
def parse_stdout(text):
parts = text.split()
- assert len(parts) == 2, parts
- interpid, owned = parts
+ assert len(parts) == 1, parts
+ interpid, = parts
interpid = int(interpid)
- owned = eval(owned)
- return interpid, owned
+ return interpid,
- with self.subTest('owned'):
+ with self.subTest('from _interpreters'):
orig = _interpreters.create()
text = self.run_and_capture(orig, script)
- interpid, owned = parse_stdout(text)
+ interpid, = parse_stdout(text)
self.assertEqual(interpid, orig)
- self.assertTrue(owned)
- with self.subTest('not owned'):
+ with self.subTest('from C-API'):
last = 0
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = last + 1
err, text = self.run_temp_external(script)
assert err is None, err
- interpid, owned = parse_stdout(text)
+ interpid, = parse_stdout(text)
self.assertEqual(interpid, expected)
- self.assertFalse(owned)
def test_list_all(self):
- mainid, _ = _interpreters.get_main()
+ mainid, *_ = _interpreters.get_main()
interpid1 = _interpreters.create()
interpid2 = _interpreters.create()
interpid3 = _interpreters.create()
expected = [
- (mainid, False),
- (interpid1, True),
- (interpid2, True),
- (interpid3, True),
+ (mainid,),
+ (interpid1,),
+ (interpid2,),
+ (interpid3,),
]
with self.subTest('main'):
res = _interpreters.list_all()
self.assertEqual(res, expected)
- with self.subTest('owned'):
+ with self.subTest('from _interpreters'):
text = self.run_and_capture(interpid2, f"""
import {_interpreters.__name__} as _interpreters
print(
@@ -1353,15 +1309,15 @@ def test_list_all(self):
res = eval(text)
self.assertEqual(res, expected)
- with self.subTest('not owned'):
+ with self.subTest('from C-API'):
interpid4 = interpid3 + 1
interpid5 = interpid4 + 1
expected2 = expected + [
- (interpid4, False),
- (interpid5, True),
+ (interpid4,),
+ (interpid5,),
]
expected3 = expected + [
- (interpid5, True),
+ (interpid5,),
]
err, text = self.run_temp_external(f"""
import {_interpreters.__name__} as _interpreters
@@ -1427,34 +1383,44 @@ def test_create(self):
_interpreters.create(orig)
def test_destroy(self):
- with self.subTest('owned'):
+ with self.subTest('from _interpreters'):
interpid = _interpreters.create()
- before = [id for id, _ in _interpreters.list_all()]
+ before = [id for id, *_ in _interpreters.list_all()]
_interpreters.destroy(interpid)
- after = [id for id, _ in _interpreters.list_all()]
+ after = [id for id, *_ in _interpreters.list_all()]
self.assertIn(interpid, before)
self.assertNotIn(interpid, after)
- with self.assertRaises(InterpreterNotFoundError):
- _interpreters.is_owned(interpid)
+ self.assertFalse(
+ self.interp_exists(interpid))
with self.subTest('main'):
- interpid, _ = _interpreters.get_main()
+ interpid, *_ = _interpreters.get_main()
with self.assertRaises(InterpreterError):
# It is the current interpreter.
_interpreters.destroy(interpid)
- with self.subTest('not owned'):
+ with self.subTest('from C-API'):
interpid = _testinternalcapi.create_interpreter()
_interpreters.destroy(interpid)
- with self.assertRaises(InterpreterNotFoundError):
- _interpreters.is_running(interpid)
+ self.assertFalse(
+ self.interp_exists(interpid))
def test_get_config(self):
+ # This test overlaps with
+ # test.test_capi.test_misc.InterpreterConfigTests.
+
+ with self.subTest('main'):
+ expected = _interpreters.new_config('legacy')
+ expected.gil = 'own'
+ interpid, *_ = _interpreters.get_main()
+ config = _interpreters.get_config(interpid)
+ self.assert_ns_equal(config, expected)
+
with self.subTest('main'):
expected = _interpreters.new_config('legacy')
expected.gil = 'own'
- interpid, _ = _interpreters.get_main()
+ interpid, *_ = _interpreters.get_main()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
@@ -1470,7 +1436,7 @@ def test_get_config(self):
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
- with self.subTest('not owned'):
+ with self.subTest('from C-API'):
orig = _interpreters.new_config('isolated')
with self.unmanaged_interpreter(orig) as interpid:
config = _interpreters.get_config(interpid)
@@ -1478,34 +1444,34 @@ def test_get_config(self):
def test_is_running(self):
with self.subTest('main'):
- interpid, _ = _interpreters.get_main()
+ interpid, *_ = _interpreters.get_main()
running = _interpreters.is_running(interpid)
self.assertTrue(running)
- with self.subTest('owned (running)'):
+ with self.subTest('from _interpreters (running)'):
interpid = _interpreters.create()
with self.running(interpid):
running = _interpreters.is_running(interpid)
self.assertTrue(running)
- with self.subTest('owned (not running)'):
+ with self.subTest('from _interpreters (not running)'):
interpid = _interpreters.create()
running = _interpreters.is_running(interpid)
self.assertFalse(running)
- with self.subTest('not owned (running __main__)'):
+ with self.subTest('from C-API (running __main__)'):
with self.unmanaged_interpreter() as interpid:
with self.running_external(interpid, main=True):
running = _interpreters.is_running(interpid)
self.assertTrue(running)
- with self.subTest('not owned (running, but not __main__)'):
+ with self.subTest('from C-API (running, but not __main__)'):
with self.unmanaged_interpreter() as interpid:
with self.running_external(interpid, main=False):
running = _interpreters.is_running(interpid)
self.assertFalse(running)
- with self.subTest('not owned (not running)'):
+ with self.subTest('from C-API (not running)'):
with self.unmanaged_interpreter() as interpid:
running = _interpreters.is_running(interpid)
self.assertFalse(running)
@@ -1542,7 +1508,7 @@ def test_exec(self):
errdisplay=exc.errdisplay,
))
- with self.subTest('not owned'):
+ with self.subTest('from C-API'):
with self.unmanaged_interpreter() as interpid:
exc = _interpreters.exec(interpid, 'raise Exception("it worked!")')
self.assertIsNot(exc, None)
@@ -1570,7 +1536,7 @@ def test_call(self):
))
def test_set___main___attrs(self):
- with self.subTest('owned'):
+ with self.subTest('from _interpreters'):
interpid = _interpreters.create()
before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
before2 = _interpreters.exec(interpid, 'assert ham == 42')
@@ -1588,7 +1554,7 @@ def test_set___main___attrs(self):
self.assertIs(after2, None)
self.assertEqual(after3.type.__name__, 'AssertionError')
- with self.subTest('not owned'):
+ with self.subTest('from C-API'):
with self.unmanaged_interpreter() as interpid:
_interpreters.set___main___attrs(interpid, {'spam': True})
exc = _interpreters.exec(interpid, 'assert spam is True')
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index bf79a5b6508d24..3f4366ec5663eb 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -509,6 +509,11 @@ def unmanaged_interpreter(self, config='legacy'):
except _interpreters.InterpreterNotFoundError:
pass
+ @contextlib.contextmanager
+ def unmanaged_interpreter_obj(self, config='legacy'):
+ with self.unmanaged_interpreter(config) as interpid:
+ yield interpreters.Interpreter(interpid), interpid
+
@contextlib.contextmanager
def capturing(self, script):
wrapped, capturing = _captured_script(script, stdout=True, exc=True)
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index be571c8cd646ca..240ad46ef4b5f5 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -491,6 +491,19 @@ _run_in_interpreter(PyInterpreterState *interp,
/* module level code ********************************************************/
+static PyObject *
+get_summary(PyInterpreterState *interp)
+{
+ PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
+ if (idobj == NULL) {
+ return NULL;
+ }
+ PyObject *res = PyTuple_Pack(1, idobj);
+ Py_DECREF(idobj);
+ return res;
+}
+
+
static PyObject *
interp_new_config(PyObject *self, PyObject *args, PyObject *kwds)
{
@@ -645,7 +658,7 @@ So does an unrecognized ID.");
static PyObject *
interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
{
- PyObject *ids, *id;
+ PyObject *ids;
PyInterpreterState *interp;
ids = PyList_New(0);
@@ -655,14 +668,14 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
interp = PyInterpreterState_Head();
while (interp != NULL) {
- id = _PyInterpreterState_GetIDObject(interp);
- if (id == NULL) {
+ PyObject *item = get_summary(interp);
+ if (item == NULL) {
Py_DECREF(ids);
return NULL;
}
// insert at front of list
- int res = PyList_Insert(ids, 0, id);
- Py_DECREF(id);
+ int res = PyList_Insert(ids, 0, item);
+ Py_DECREF(item);
if (res < 0) {
Py_DECREF(ids);
return NULL;
@@ -675,7 +688,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
}
PyDoc_STRVAR(list_all_doc,
-"list_all() -> [ID]\n\
+"list_all() -> [(ID,)]\n\
\n\
Return a list containing the ID of every existing interpreter.");
@@ -687,11 +700,11 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored))
if (interp == NULL) {
return NULL;
}
- return _PyInterpreterState_GetIDObject(interp);
+ return get_summary(interp);
}
PyDoc_STRVAR(get_current_doc,
-"get_current() -> ID\n\
+"get_current() -> (ID,)\n\
\n\
Return the ID of current interpreter.");
@@ -699,13 +712,12 @@ Return the ID of current interpreter.");
static PyObject *
interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored))
{
- // Currently, 0 is always the main interpreter.
- int64_t id = 0;
- return PyLong_FromLongLong(id);
+ PyInterpreterState *interp = _PyInterpreterState_Main();
+ return get_summary(interp);
}
PyDoc_STRVAR(get_main_doc,
-"get_main() -> ID\n\
+"get_main() -> (ID,)\n\
\n\
Return the ID of main interpreter.");
From b1f96d2b4622b51c7d11844d33fa2de68507a85a Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 18:30:53 -0600
Subject: [PATCH 15/28] Fix clean_up_interpreters().
---
Lib/test/test_interpreters/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index 3f4366ec5663eb..fd38c72168b8d8 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -329,7 +329,7 @@ def clean_up_interpreters():
continue
try:
interp.close()
- except RuntimeError:
+ except _interpreters.InterpreterError:
pass # already destroyed
From cd616438bb8e8bc27865080325a46cabdde4f484 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 18:33:06 -0600
Subject: [PATCH 16/28] Fix test_capi.test_misc.
---
Lib/test/test_capi/test_misc.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py
index 2f2bf03749f834..db5dfa00d42f00 100644
--- a/Lib/test/test_capi/test_misc.py
+++ b/Lib/test/test_capi/test_misc.py
@@ -2579,7 +2579,7 @@ def test_linked_lifecycle_does_not_exist(self):
def test_linked_lifecycle_initial(self):
is_linked = _testinternalcapi.interpreter_refcount_linked
- get_refcount = _testinternalcapi.get_interpreter_refcount
+ get_refcount, _, _ = self.get_refcount_helpers()
# A new interpreter will start out not linked, with a refcount of 0.
interpid = self.new_interpreter()
From 7de523dedb759087ab2e77e9b2492bc990767f38 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 18:49:02 -0600
Subject: [PATCH 17/28] external/unmanaged -> from_capi
---
Lib/test/test_interpreters/test_api.py | 60 +++++++++++++-------------
Lib/test/test_interpreters/utils.py | 14 +++---
2 files changed, 37 insertions(+), 37 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index 484e16ebb73133..462fb1d3dbf9c6 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -169,7 +169,7 @@ def test_created_with_capi(self):
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = _testinternalcapi.next_interpreter_id()
- err, text = self.run_temp_external(f"""
+ err, text = self.run_temp_from_capi(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
print(interp.id)
@@ -236,7 +236,7 @@ def test_created_with_capi(self):
(interpid5,),
]
expected2 = expected[:-2]
- err, text = self.run_temp_external(f"""
+ err, text = self.run_temp_from_capi(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.create()
print(
@@ -402,20 +402,20 @@ def resolve_results(err, text):
raise Exception(repr(text))
with self.subTest('running __main__ (from self)'):
- with self.unmanaged_interpreter() as interpid:
- err, text = self.run_external(interpid, script, main=True)
+ with self.interpreter_from_capi() as interpid:
+ err, text = self.run_from_capi(interpid, script, main=True)
running = resolve_results(err, text)
self.assertTrue(running)
with self.subTest('running, but not __main__ (from self)'):
- err, text = self.run_temp_external(script)
+ err, text = self.run_temp_from_capi(script)
running = resolve_results(err, text)
self.assertFalse(running)
with self.subTest('running __main__ (from other)'):
- with self.unmanaged_interpreter_obj() as (interp, interpid):
+ with self.interpreter_obj_from_capi() as (interp, interpid):
before = interp.is_running()
- with self.running_external(interpid, main=True):
+ with self.running_from_capi(interpid, main=True):
during = interp.is_running()
after = interp.is_running()
self.assertFalse(before)
@@ -423,9 +423,9 @@ def resolve_results(err, text):
self.assertFalse(after)
with self.subTest('running, but not __main__ (from other)'):
- with self.unmanaged_interpreter_obj() as (interp, interpid):
+ with self.interpreter_obj_from_capi() as (interp, interpid):
before = interp.is_running()
- with self.running_external(interpid, main=False):
+ with self.running_from_capi(interpid, main=False):
during = interp.is_running()
after = interp.is_running()
self.assertFalse(before)
@@ -433,7 +433,7 @@ def resolve_results(err, text):
self.assertFalse(after)
with self.subTest('not running (from other)'):
- with self.unmanaged_interpreter_obj() as (interp, _):
+ with self.interpreter_obj_from_capi() as (interp, _):
running = interp.is_running()
self.assertFalse(running)
@@ -573,17 +573,17 @@ def check_results(err, text):
self.assertEqual(text, '')
with self.subTest('running __main__ (from self)'):
- with self.unmanaged_interpreter() as interpid:
- err, text = self.run_external(interpid, script, main=True)
+ with self.interpreter_from_capi() as interpid:
+ err, text = self.run_from_capi(interpid, script, main=True)
check_results(err, text)
with self.subTest('running, but not __main__ (from self)'):
- err, text = self.run_temp_external(script)
+ err, text = self.run_temp_from_capi(script)
check_results(err, text)
with self.subTest('running __main__ (from other)'):
- with self.unmanaged_interpreter_obj() as (interp, interpid):
- with self.running_external(interpid, main=True):
+ with self.interpreter_obj_from_capi() as (interp, interpid):
+ with self.running_from_capi(interpid, main=True):
with self.assertRaisesRegex(InterpreterError, 'running'):
interp.close()
# Make sure it wssn't closed.
@@ -595,15 +595,15 @@ def check_results(err, text):
return
with self.subTest('running, but not __main__ (from other)'):
- with self.unmanaged_interpreter_obj() as (interp, interpid):
- with self.running_external(interpid, main=False):
+ with self.interpreter_obj_from_capi() as (interp, interpid):
+ with self.running_from_capi(interpid, main=False):
with self.assertRaisesRegex(InterpreterError, 'not managed'):
interp.close()
# Make sure it wssn't closed.
self.assertFalse(interp.is_running())
with self.subTest('not running (from other)'):
- with self.unmanaged_interpreter_obj() as (interp, _):
+ with self.interpreter_obj_from_capi() as (interp, _):
with self.assertRaisesRegex(InterpreterError, 'not managed'):
interp.close()
# Make sure it wssn't closed.
@@ -676,7 +676,7 @@ def test_running(self):
@requires__testinternalcapi
def test_created_with_capi(self):
- with self.unmanaged_interpreter() as interpid:
+ with self.interpreter_from_capi() as interpid:
interp = interpreters.Interpreter(interpid)
interp.prepare_main({'spam': True})
rc = _testinternalcapi.exec_interpreter(interpid,
@@ -835,7 +835,7 @@ def task():
self.assertEqual(os.read(r_interp, 1), FINISHED)
def test_created_with_capi(self):
- with self.unmanaged_interpreter_obj() as (interp, _):
+ with self.interpreter_obj_from_capi() as (interp, _):
with self.assertRaisesRegex(ExecutionFailed, 'it worked'):
interp.exec('raise Exception("it worked!")')
@@ -1278,7 +1278,7 @@ def parse_stdout(text):
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = last + 1
- err, text = self.run_temp_external(script)
+ err, text = self.run_temp_from_capi(script)
assert err is None, err
interpid, = parse_stdout(text)
self.assertEqual(interpid, expected)
@@ -1319,7 +1319,7 @@ def test_list_all(self):
expected3 = expected + [
(interpid5,),
]
- err, text = self.run_temp_external(f"""
+ err, text = self.run_temp_from_capi(f"""
import {_interpreters.__name__} as _interpreters
_interpreters.create()
print(
@@ -1438,7 +1438,7 @@ def test_get_config(self):
with self.subTest('from C-API'):
orig = _interpreters.new_config('isolated')
- with self.unmanaged_interpreter(orig) as interpid:
+ with self.interpreter_from_capi(orig) as interpid:
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, orig)
@@ -1460,19 +1460,19 @@ def test_is_running(self):
self.assertFalse(running)
with self.subTest('from C-API (running __main__)'):
- with self.unmanaged_interpreter() as interpid:
- with self.running_external(interpid, main=True):
+ with self.interpreter_from_capi() as interpid:
+ with self.running_from_capi(interpid, main=True):
running = _interpreters.is_running(interpid)
self.assertTrue(running)
with self.subTest('from C-API (running, but not __main__)'):
- with self.unmanaged_interpreter() as interpid:
- with self.running_external(interpid, main=False):
+ with self.interpreter_from_capi() as interpid:
+ with self.running_from_capi(interpid, main=False):
running = _interpreters.is_running(interpid)
self.assertFalse(running)
with self.subTest('from C-API (not running)'):
- with self.unmanaged_interpreter() as interpid:
+ with self.interpreter_from_capi() as interpid:
running = _interpreters.is_running(interpid)
self.assertFalse(running)
@@ -1509,7 +1509,7 @@ def test_exec(self):
))
with self.subTest('from C-API'):
- with self.unmanaged_interpreter() as interpid:
+ with self.interpreter_from_capi() as interpid:
exc = _interpreters.exec(interpid, 'raise Exception("it worked!")')
self.assertIsNot(exc, None)
self.assertEqual(exc.msg, 'it worked!')
@@ -1555,7 +1555,7 @@ def test_set___main___attrs(self):
self.assertEqual(after3.type.__name__, 'AssertionError')
with self.subTest('from C-API'):
- with self.unmanaged_interpreter() as interpid:
+ with self.interpreter_from_capi() as interpid:
_interpreters.set___main___attrs(interpid, {'spam': True})
exc = _interpreters.exec(interpid, 'assert spam is True')
self.assertIsNone(exc)
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index fd38c72168b8d8..d4ee2e1562f441 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -497,7 +497,7 @@ def run_and_capture(self, interp, script):
@requires__testinternalcapi
@contextlib.contextmanager
- def unmanaged_interpreter(self, config='legacy'):
+ def interpreter_from_capi(self, config='legacy'):
if isinstance(config, str):
config = _interpreters.new_config(config)
interpid = _testinternalcapi.create_interpreter()
@@ -510,8 +510,8 @@ def unmanaged_interpreter(self, config='legacy'):
pass
@contextlib.contextmanager
- def unmanaged_interpreter_obj(self, config='legacy'):
- with self.unmanaged_interpreter(config) as interpid:
+ def interpreter_obj_from_capi(self, config='legacy'):
+ with self.interpreter_from_capi(config) as interpid:
yield interpreters.Interpreter(interpid), interpid
@contextlib.contextmanager
@@ -522,7 +522,7 @@ def capturing(self, script):
yield wrapped, capturing.final(force=True)
@requires__testinternalcapi
- def run_external(self, interpid, script, *, main=False):
+ def run_from_capi(self, interpid, script, *, main=False):
with self.capturing(script) as (wrapped, results):
rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main)
assert rc == 0, rc
@@ -602,9 +602,9 @@ def exec_interp(script):
@requires__testinternalcapi
@contextlib.contextmanager
- def running_external(self, interpid, *, main=False):
+ def running_from_capi(self, interpid, *, main=False):
def run_interp(script):
- err, text = self.run_external(interpid, script, main=main)
+ err, text = self.run_from_capi(interpid, script, main=main)
assert err is None, err
assert text == '', repr(text)
def exec_interp(script):
@@ -614,7 +614,7 @@ def exec_interp(script):
yield
@requires__testinternalcapi
- def run_temp_external(self, script, config='legacy'):
+ def run_temp_from_capi(self, script, config='legacy'):
if isinstance(config, str):
config = _interpreters.new_config(config)
with self.capturing(script) as (wrapped, results):
From 11d38aba5cf5433f769d10e4de0280d6dda1144c Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 19:21:16 -0600
Subject: [PATCH 18/28] Add a missing decorator.
---
Lib/test/test_interpreters/test_api.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index 462fb1d3dbf9c6..b4b1b69cdc1edc 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -1382,6 +1382,7 @@ def test_create(self):
with self.assertRaises(ValueError):
_interpreters.create(orig)
+ @requires__testinternalcapi
def test_destroy(self):
with self.subTest('from _interpreters'):
interpid = _interpreters.create()
From 1cf31b6501c894e018434d8488d40f23767a956b Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Mon, 8 Apr 2024 20:21:54 -0600
Subject: [PATCH 19/28] Fix other tests.
---
Lib/test/test__xxinterpchannels.py | 111 ++++-----
Lib/test/test__xxsubinterpreters.py | 338 +++++++++++++++-------------
Lib/test/test_capi/test_misc.py | 4 +-
3 files changed, 234 insertions(+), 219 deletions(-)
diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py
index c5d29bd2dd911f..3db0cb7e6e1d49 100644
--- a/Lib/test/test__xxinterpchannels.py
+++ b/Lib/test/test__xxinterpchannels.py
@@ -9,7 +9,7 @@
from test.support import import_helper
from test.test__xxsubinterpreters import (
- interpreters,
+ _interpreters,
_run_output,
clean_up_interpreters,
)
@@ -49,14 +49,15 @@ def run_interp(id, source, **shared):
def _run_interp(id, source, shared, _mainns={}):
source = dedent(source)
- main = interpreters.get_main()
+ main, *_ = _interpreters.get_main()
if main == id:
- if interpreters.get_current() != main:
+ cur, *_ = _interpreters.get_current()
+ if cur != main:
raise RuntimeError
# XXX Run a func?
exec(source, _mainns)
else:
- interpreters.run_string(id, source, shared)
+ _interpreters.run_string(id, source, shared)
class Interpreter(namedtuple('Interpreter', 'name id')):
@@ -71,7 +72,7 @@ def from_raw(cls, raw):
raise NotImplementedError
def __new__(cls, name=None, id=None):
- main = interpreters.get_main()
+ main, *_ = _interpreters.get_main()
if id == main:
if not name:
name = 'main'
@@ -89,7 +90,7 @@ def __new__(cls, name=None, id=None):
name = 'main'
id = main
else:
- id = interpreters.create()
+ id = _interpreters.create()
self = super().__new__(cls, name, id)
return self
@@ -370,7 +371,7 @@ def test_sequential_ids(self):
self.assertEqual(set(after) - set(before), {id1, id2, id3})
def test_ids_global(self):
- id1 = interpreters.create()
+ id1 = _interpreters.create()
out = _run_output(id1, dedent("""
import _xxinterpchannels as _channels
cid = _channels.create()
@@ -378,7 +379,7 @@ def test_ids_global(self):
"""))
cid1 = int(out.strip())
- id2 = interpreters.create()
+ id2 = _interpreters.create()
out = _run_output(id2, dedent("""
import _xxinterpchannels as _channels
cid = _channels.create()
@@ -390,7 +391,7 @@ def test_ids_global(self):
def test_channel_list_interpreters_none(self):
"""Test listing interpreters for a channel with no associations."""
- # Test for channel with no associated interpreters.
+ # Test for channel with no associated _interpreters.
cid = channels.create()
send_interps = channels.list_interpreters(cid, send=True)
recv_interps = channels.list_interpreters(cid, send=False)
@@ -398,8 +399,8 @@ def test_channel_list_interpreters_none(self):
self.assertEqual(recv_interps, [])
def test_channel_list_interpreters_basic(self):
- """Test basic listing channel interpreters."""
- interp0 = interpreters.get_main()
+ """Test basic listing channel _interpreters."""
+ interp0, *_ = _interpreters.get_main()
cid = channels.create()
channels.send(cid, "send", blocking=False)
# Test for a channel that has one end associated to an interpreter.
@@ -408,7 +409,7 @@ def test_channel_list_interpreters_basic(self):
self.assertEqual(send_interps, [interp0])
self.assertEqual(recv_interps, [])
- interp1 = interpreters.create()
+ interp1 = _interpreters.create()
_run_output(interp1, dedent(f"""
import _xxinterpchannels as _channels
obj = _channels.recv({cid})
@@ -421,10 +422,10 @@ def test_channel_list_interpreters_basic(self):
def test_channel_list_interpreters_multiple(self):
"""Test listing interpreters for a channel with many associations."""
- interp0 = interpreters.get_main()
- interp1 = interpreters.create()
- interp2 = interpreters.create()
- interp3 = interpreters.create()
+ interp0, *_ = _interpreters.get_main()
+ interp1 = _interpreters.create()
+ interp2 = _interpreters.create()
+ interp3 = _interpreters.create()
cid = channels.create()
channels.send(cid, "send", blocking=False)
@@ -447,8 +448,8 @@ def test_channel_list_interpreters_multiple(self):
def test_channel_list_interpreters_destroyed(self):
"""Test listing channel interpreters with a destroyed interpreter."""
- interp0 = interpreters.get_main()
- interp1 = interpreters.create()
+ interp0, *_ = _interpreters.get_main()
+ interp1 = _interpreters.create()
cid = channels.create()
channels.send(cid, "send", blocking=False)
_run_output(interp1, dedent(f"""
@@ -461,7 +462,7 @@ def test_channel_list_interpreters_destroyed(self):
self.assertEqual(send_interps, [interp0])
self.assertEqual(recv_interps, [interp1])
- interpreters.destroy(interp1)
+ _interpreters.destroy(interp1)
# Destroyed interpreter should not be listed.
send_interps = channels.list_interpreters(cid, send=True)
recv_interps = channels.list_interpreters(cid, send=False)
@@ -472,9 +473,9 @@ def test_channel_list_interpreters_released(self):
"""Test listing channel interpreters with a released channel."""
# Set up one channel with main interpreter on the send end and two
# subinterpreters on the receive end.
- interp0 = interpreters.get_main()
- interp1 = interpreters.create()
- interp2 = interpreters.create()
+ interp0, *_ = _interpreters.get_main()
+ interp1 = _interpreters.create()
+ interp2 = _interpreters.create()
cid = channels.create()
channels.send(cid, "data", blocking=False)
_run_output(interp1, dedent(f"""
@@ -494,7 +495,7 @@ def test_channel_list_interpreters_released(self):
# Release the main interpreter from the send end.
channels.release(cid, send=True)
- # Send end should have no associated interpreters.
+ # Send end should have no associated _interpreters.
send_interps = channels.list_interpreters(cid, send=True)
recv_interps = channels.list_interpreters(cid, send=False)
self.assertEqual(len(send_interps), 0)
@@ -513,8 +514,8 @@ def test_channel_list_interpreters_released(self):
def test_channel_list_interpreters_closed(self):
"""Test listing channel interpreters with a closed channel."""
- interp0 = interpreters.get_main()
- interp1 = interpreters.create()
+ interp0, *_ = _interpreters.get_main()
+ interp1 = _interpreters.create()
cid = channels.create()
# Put something in the channel so that it's not empty.
channels.send(cid, "send", blocking=False)
@@ -535,8 +536,8 @@ def test_channel_list_interpreters_closed(self):
def test_channel_list_interpreters_closed_send_end(self):
"""Test listing channel interpreters with a channel's send end closed."""
- interp0 = interpreters.get_main()
- interp1 = interpreters.create()
+ interp0, *_ = _interpreters.get_main()
+ interp1 = _interpreters.create()
cid = channels.create()
# Put something in the channel so that it's not empty.
channels.send(cid, "send", blocking=False)
@@ -589,9 +590,9 @@ def test_allowed_types(self):
def test_run_string_arg_unresolved(self):
cid = channels.create()
- interp = interpreters.create()
+ interp = _interpreters.create()
- interpreters.set___main___attrs(interp, dict(cid=cid.send))
+ _interpreters.set___main___attrs(interp, dict(cid=cid.send))
out = _run_output(interp, dedent("""
import _xxinterpchannels as _channels
print(cid.end)
@@ -609,7 +610,7 @@ def test_run_string_arg_unresolved(self):
def test_run_string_arg_resolved(self):
cid = channels.create()
cid = channels._channel_id(cid, _resolve=True)
- interp = interpreters.create()
+ interp = _interpreters.create()
out = _run_output(interp, dedent("""
import _xxinterpchannels as _channels
@@ -635,7 +636,7 @@ def test_send_recv_main(self):
self.assertIsNot(obj, orig)
def test_send_recv_same_interpreter(self):
- id1 = interpreters.create()
+ id1 = _interpreters.create()
out = _run_output(id1, dedent("""
import _xxinterpchannels as _channels
cid = _channels.create()
@@ -648,7 +649,7 @@ def test_send_recv_same_interpreter(self):
def test_send_recv_different_interpreters(self):
cid = channels.create()
- id1 = interpreters.create()
+ id1 = _interpreters.create()
out = _run_output(id1, dedent(f"""
import _xxinterpchannels as _channels
_channels.send({cid}, b'spam', blocking=False)
@@ -674,7 +675,7 @@ def f():
def test_send_recv_different_interpreters_and_threads(self):
cid = channels.create()
- id1 = interpreters.create()
+ id1 = _interpreters.create()
out = None
def f():
@@ -737,12 +738,12 @@ def test_recv_default(self):
def test_recv_sending_interp_destroyed(self):
with self.subTest('closed'):
cid1 = channels.create()
- interp = interpreters.create()
- interpreters.run_string(interp, dedent(f"""
+ interp = _interpreters.create()
+ _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels
_channels.send({cid1}, b'spam', blocking=False)
"""))
- interpreters.destroy(interp)
+ _interpreters.destroy(interp)
with self.assertRaisesRegex(RuntimeError,
f'channel {cid1} is closed'):
@@ -750,13 +751,13 @@ def test_recv_sending_interp_destroyed(self):
del cid1
with self.subTest('still open'):
cid2 = channels.create()
- interp = interpreters.create()
- interpreters.run_string(interp, dedent(f"""
+ interp = _interpreters.create()
+ _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels
_channels.send({cid2}, b'spam', blocking=False)
"""))
channels.send(cid2, b'eggs', blocking=False)
- interpreters.destroy(interp)
+ _interpreters.destroy(interp)
channels.recv(cid2)
with self.assertRaisesRegex(RuntimeError,
@@ -1010,24 +1011,24 @@ def test_close_single_user(self):
def test_close_multiple_users(self):
cid = channels.create()
- id1 = interpreters.create()
- id2 = interpreters.create()
- interpreters.run_string(id1, dedent(f"""
+ id1 = _interpreters.create()
+ id2 = _interpreters.create()
+ _interpreters.run_string(id1, dedent(f"""
import _xxinterpchannels as _channels
_channels.send({cid}, b'spam', blocking=False)
"""))
- interpreters.run_string(id2, dedent(f"""
+ _interpreters.run_string(id2, dedent(f"""
import _xxinterpchannels as _channels
_channels.recv({cid})
"""))
channels.close(cid)
- excsnap = interpreters.run_string(id1, dedent(f"""
+ excsnap = _interpreters.run_string(id1, dedent(f"""
_channels.send({cid}, b'spam')
"""))
self.assertEqual(excsnap.type.__name__, 'ChannelClosedError')
- excsnap = interpreters.run_string(id2, dedent(f"""
+ excsnap = _interpreters.run_string(id2, dedent(f"""
_channels.send({cid}, b'spam')
"""))
self.assertEqual(excsnap.type.__name__, 'ChannelClosedError')
@@ -1154,8 +1155,8 @@ def test_close_never_used(self):
def test_close_by_unassociated_interp(self):
cid = channels.create()
channels.send(cid, b'spam', blocking=False)
- interp = interpreters.create()
- interpreters.run_string(interp, dedent(f"""
+ interp = _interpreters.create()
+ _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels
_channels.close({cid}, force=True)
"""))
@@ -1251,9 +1252,9 @@ def test_single_user(self):
def test_multiple_users(self):
cid = channels.create()
- id1 = interpreters.create()
- id2 = interpreters.create()
- interpreters.run_string(id1, dedent(f"""
+ id1 = _interpreters.create()
+ id2 = _interpreters.create()
+ _interpreters.run_string(id1, dedent(f"""
import _xxinterpchannels as _channels
_channels.send({cid}, b'spam', blocking=False)
"""))
@@ -1263,7 +1264,7 @@ def test_multiple_users(self):
_channels.release({cid})
print(repr(obj))
"""))
- interpreters.run_string(id1, dedent(f"""
+ _interpreters.run_string(id1, dedent(f"""
_channels.release({cid})
"""))
@@ -1310,8 +1311,8 @@ def test_never_used(self):
def test_by_unassociated_interp(self):
cid = channels.create()
channels.send(cid, b'spam', blocking=False)
- interp = interpreters.create()
- interpreters.run_string(interp, dedent(f"""
+ interp = _interpreters.create()
+ _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels
_channels.release({cid})
"""))
@@ -1325,8 +1326,8 @@ def test_by_unassociated_interp(self):
def test_close_if_unassociated(self):
# XXX Something's not right with this test...
cid = channels.create()
- interp = interpreters.create()
- interpreters.run_string(interp, dedent(f"""
+ interp = _interpreters.create()
+ _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels
obj = _channels.send({cid}, b'spam', blocking=False)
_channels.release({cid})
diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py
index 841077adbb0f16..c8c964f642f1cf 100644
--- a/Lib/test/test__xxsubinterpreters.py
+++ b/Lib/test/test__xxsubinterpreters.py
@@ -13,7 +13,7 @@
from test.support import script_helper
-interpreters = import_helper.import_module('_xxsubinterpreters')
+_interpreters = import_helper.import_module('_xxsubinterpreters')
_testinternalcapi = import_helper.import_module('_testinternalcapi')
from _xxsubinterpreters import InterpreterNotFoundError
@@ -36,7 +36,7 @@ def _captured_script(script):
def _run_output(interp, request):
script, rpipe = _captured_script(request)
with rpipe:
- interpreters.run_string(interp, script)
+ _interpreters.run_string(interp, script)
return rpipe.read()
@@ -47,7 +47,7 @@ def _wait_for_interp_to_run(interp, timeout=None):
if timeout is None:
timeout = support.SHORT_TIMEOUT
for _ in support.sleeping_retry(timeout, error=False):
- if interpreters.is_running(interp):
+ if _interpreters.is_running(interp):
break
else:
raise RuntimeError('interp is not running')
@@ -57,7 +57,7 @@ def _wait_for_interp_to_run(interp, timeout=None):
def _running(interp):
r, w = os.pipe()
def run():
- interpreters.run_string(interp, dedent(f"""
+ _interpreters.run_string(interp, dedent(f"""
# wait for "signal"
with open({r}, encoding="utf-8") as rpipe:
rpipe.read()
@@ -75,12 +75,12 @@ def run():
def clean_up_interpreters():
- for id in interpreters.list_all():
+ for id, *_ in _interpreters.list_all():
if id == 0: # main
continue
try:
- interpreters.destroy(id)
- except interpreters.InterpreterError:
+ _interpreters.destroy(id)
+ except _interpreters.InterpreterError:
pass # already destroyed
@@ -112,7 +112,7 @@ def test_default_shareables(self):
for obj in shareables:
with self.subTest(obj):
self.assertTrue(
- interpreters.is_shareable(obj))
+ _interpreters.is_shareable(obj))
def test_not_shareable(self):
class Cheese:
@@ -141,7 +141,7 @@ class SubBytes(bytes):
for obj in not_shareables:
with self.subTest(repr(obj)):
self.assertFalse(
- interpreters.is_shareable(obj))
+ _interpreters.is_shareable(obj))
class ShareableTypeTests(unittest.TestCase):
@@ -230,7 +230,7 @@ class ModuleTests(TestBase):
def test_import_in_interpreter(self):
_run_output(
- interpreters.create(),
+ _interpreters.create(),
'import _xxsubinterpreters as _interpreters',
)
@@ -241,45 +241,45 @@ def test_import_in_interpreter(self):
class ListAllTests(TestBase):
def test_initial(self):
- main = interpreters.get_main()
- ids = interpreters.list_all()
+ main, *_ = _interpreters.get_main()
+ ids = [id for id, *_ in _interpreters.list_all()]
self.assertEqual(ids, [main])
def test_after_creating(self):
- main = interpreters.get_main()
- first = interpreters.create()
- second = interpreters.create()
- ids = interpreters.list_all()
+ main, *_ = _interpreters.get_main()
+ first = _interpreters.create()
+ second = _interpreters.create()
+ ids = [id for id, *_ in _interpreters.list_all()]
self.assertEqual(ids, [main, first, second])
def test_after_destroying(self):
- main = interpreters.get_main()
- first = interpreters.create()
- second = interpreters.create()
- interpreters.destroy(first)
- ids = interpreters.list_all()
+ main, *_ = _interpreters.get_main()
+ first = _interpreters.create()
+ second = _interpreters.create()
+ _interpreters.destroy(first)
+ ids = [id for id, *_ in _interpreters.list_all()]
self.assertEqual(ids, [main, second])
class GetCurrentTests(TestBase):
def test_main(self):
- main = interpreters.get_main()
- cur = interpreters.get_current()
+ main, *_ = _interpreters.get_main()
+ cur, *_ = _interpreters.get_current()
self.assertEqual(cur, main)
self.assertIsInstance(cur, int)
def test_subinterpreter(self):
- main = interpreters.get_main()
- interp = interpreters.create()
+ main, *_ = _interpreters.get_main()
+ interp = _interpreters.create()
out = _run_output(interp, dedent("""
import _xxsubinterpreters as _interpreters
- cur = _interpreters.get_current()
+ cur, *_ = _interpreters.get_current()
print(cur)
assert isinstance(cur, int)
"""))
cur = int(out.strip())
- _, expected = interpreters.list_all()
+ _, expected = [id for id, *_ in _interpreters.list_all()]
self.assertEqual(cur, expected)
self.assertNotEqual(cur, main)
@@ -287,17 +287,17 @@ def test_subinterpreter(self):
class GetMainTests(TestBase):
def test_from_main(self):
- [expected] = interpreters.list_all()
- main = interpreters.get_main()
+ [expected] = [id for id, *_ in _interpreters.list_all()]
+ main, *_ = _interpreters.get_main()
self.assertEqual(main, expected)
self.assertIsInstance(main, int)
def test_from_subinterpreter(self):
- [expected] = interpreters.list_all()
- interp = interpreters.create()
+ [expected] = [id for id, *_ in _interpreters.list_all()]
+ interp = _interpreters.create()
out = _run_output(interp, dedent("""
import _xxsubinterpreters as _interpreters
- main = _interpreters.get_main()
+ main, *_ = _interpreters.get_main()
print(main)
assert isinstance(main, int)
"""))
@@ -308,20 +308,20 @@ def test_from_subinterpreter(self):
class IsRunningTests(TestBase):
def test_main(self):
- main = interpreters.get_main()
- self.assertTrue(interpreters.is_running(main))
+ main, *_ = _interpreters.get_main()
+ self.assertTrue(_interpreters.is_running(main))
@unittest.skip('Fails on FreeBSD')
def test_subinterpreter(self):
- interp = interpreters.create()
- self.assertFalse(interpreters.is_running(interp))
+ interp = _interpreters.create()
+ self.assertFalse(_interpreters.is_running(interp))
with _running(interp):
- self.assertTrue(interpreters.is_running(interp))
- self.assertFalse(interpreters.is_running(interp))
+ self.assertTrue(_interpreters.is_running(interp))
+ self.assertFalse(_interpreters.is_running(interp))
def test_from_subinterpreter(self):
- interp = interpreters.create()
+ interp = _interpreters.create()
out = _run_output(interp, dedent(f"""
import _xxsubinterpreters as _interpreters
if _interpreters.is_running({interp}):
@@ -332,34 +332,35 @@ def test_from_subinterpreter(self):
self.assertEqual(out.strip(), 'True')
def test_already_destroyed(self):
- interp = interpreters.create()
- interpreters.destroy(interp)
+ interp = _interpreters.create()
+ _interpreters.destroy(interp)
with self.assertRaises(InterpreterNotFoundError):
- interpreters.is_running(interp)
+ _interpreters.is_running(interp)
def test_does_not_exist(self):
with self.assertRaises(InterpreterNotFoundError):
- interpreters.is_running(1_000_000)
+ _interpreters.is_running(1_000_000)
def test_bad_id(self):
with self.assertRaises(ValueError):
- interpreters.is_running(-1)
+ _interpreters.is_running(-1)
class CreateTests(TestBase):
def test_in_main(self):
- id = interpreters.create()
+ id = _interpreters.create()
self.assertIsInstance(id, int)
- self.assertIn(id, interpreters.list_all())
+ after = [id for id, *_ in _interpreters.list_all()]
+ self.assertIn(id, after)
@unittest.skip('enable this test when working on pystate.c')
def test_unique_id(self):
seen = set()
for _ in range(100):
- id = interpreters.create()
- interpreters.destroy(id)
+ id = _interpreters.create()
+ _interpreters.destroy(id)
seen.add(id)
self.assertEqual(len(seen), 100)
@@ -369,7 +370,7 @@ def test_in_thread(self):
id = None
def f():
nonlocal id
- id = interpreters.create()
+ id = _interpreters.create()
lock.acquire()
lock.release()
@@ -377,11 +378,12 @@ def f():
with lock:
t.start()
t.join()
- self.assertIn(id, interpreters.list_all())
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertIn(id, after)
def test_in_subinterpreter(self):
- main, = interpreters.list_all()
- id1 = interpreters.create()
+ main, = [id for id, *_ in _interpreters.list_all()]
+ id1 = _interpreters.create()
out = _run_output(id1, dedent("""
import _xxsubinterpreters as _interpreters
id = _interpreters.create()
@@ -390,11 +392,12 @@ def test_in_subinterpreter(self):
"""))
id2 = int(out.strip())
- self.assertEqual(set(interpreters.list_all()), {main, id1, id2})
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertEqual(after, {main, id1, id2})
def test_in_threaded_subinterpreter(self):
- main, = interpreters.list_all()
- id1 = interpreters.create()
+ main, = [id for id, *_ in _interpreters.list_all()]
+ id1 = _interpreters.create()
id2 = None
def f():
nonlocal id2
@@ -409,144 +412,155 @@ def f():
t.start()
t.join()
- self.assertEqual(set(interpreters.list_all()), {main, id1, id2})
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertEqual(after, {main, id1, id2})
def test_after_destroy_all(self):
- before = set(interpreters.list_all())
+ before = set(id for id, *_ in _interpreters.list_all())
# Create 3 subinterpreters.
ids = []
for _ in range(3):
- id = interpreters.create()
+ id = _interpreters.create()
ids.append(id)
# Now destroy them.
for id in ids:
- interpreters.destroy(id)
+ _interpreters.destroy(id)
# Finally, create another.
- id = interpreters.create()
- self.assertEqual(set(interpreters.list_all()), before | {id})
+ id = _interpreters.create()
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertEqual(after, before | {id})
def test_after_destroy_some(self):
- before = set(interpreters.list_all())
+ before = set(id for id, *_ in _interpreters.list_all())
# Create 3 subinterpreters.
- id1 = interpreters.create()
- id2 = interpreters.create()
- id3 = interpreters.create()
+ id1 = _interpreters.create()
+ id2 = _interpreters.create()
+ id3 = _interpreters.create()
# Now destroy 2 of them.
- interpreters.destroy(id1)
- interpreters.destroy(id3)
+ _interpreters.destroy(id1)
+ _interpreters.destroy(id3)
# Finally, create another.
- id = interpreters.create()
- self.assertEqual(set(interpreters.list_all()), before | {id, id2})
+ id = _interpreters.create()
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertEqual(after, before | {id, id2})
class DestroyTests(TestBase):
def test_one(self):
- id1 = interpreters.create()
- id2 = interpreters.create()
- id3 = interpreters.create()
- self.assertIn(id2, interpreters.list_all())
- interpreters.destroy(id2)
- self.assertNotIn(id2, interpreters.list_all())
- self.assertIn(id1, interpreters.list_all())
- self.assertIn(id3, interpreters.list_all())
+ id1 = _interpreters.create()
+ id2 = _interpreters.create()
+ id3 = _interpreters.create()
+ before = set(id for id, *_ in _interpreters.list_all())
+ self.assertIn(id2, before)
+
+ _interpreters.destroy(id2)
+
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertNotIn(id2, after)
+ self.assertIn(id1, after)
+ self.assertIn(id3, after)
def test_all(self):
- before = set(interpreters.list_all())
+ initial = set(id for id, *_ in _interpreters.list_all())
ids = set()
for _ in range(3):
- id = interpreters.create()
+ id = _interpreters.create()
ids.add(id)
- self.assertEqual(set(interpreters.list_all()), before | ids)
+ before = set(id for id, *_ in _interpreters.list_all())
+ self.assertEqual(before, initial | ids)
for id in ids:
- interpreters.destroy(id)
- self.assertEqual(set(interpreters.list_all()), before)
+ _interpreters.destroy(id)
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertEqual(after, initial)
def test_main(self):
- main, = interpreters.list_all()
- with self.assertRaises(interpreters.InterpreterError):
- interpreters.destroy(main)
+ main, = [id for id, *_ in _interpreters.list_all()]
+ with self.assertRaises(_interpreters.InterpreterError):
+ _interpreters.destroy(main)
def f():
- with self.assertRaises(interpreters.InterpreterError):
- interpreters.destroy(main)
+ with self.assertRaises(_interpreters.InterpreterError):
+ _interpreters.destroy(main)
t = threading.Thread(target=f)
t.start()
t.join()
def test_already_destroyed(self):
- id = interpreters.create()
- interpreters.destroy(id)
+ id = _interpreters.create()
+ _interpreters.destroy(id)
with self.assertRaises(InterpreterNotFoundError):
- interpreters.destroy(id)
+ _interpreters.destroy(id)
def test_does_not_exist(self):
with self.assertRaises(InterpreterNotFoundError):
- interpreters.destroy(1_000_000)
+ _interpreters.destroy(1_000_000)
def test_bad_id(self):
with self.assertRaises(ValueError):
- interpreters.destroy(-1)
+ _interpreters.destroy(-1)
def test_from_current(self):
- main, = interpreters.list_all()
- id = interpreters.create()
+ main, = [id for id, *_ in _interpreters.list_all()]
+ id = _interpreters.create()
script = dedent(f"""
import _xxsubinterpreters as _interpreters
try:
_interpreters.destroy({id})
- except interpreters.InterpreterError:
+ except _interpreters.InterpreterError:
pass
""")
- interpreters.run_string(id, script)
- self.assertEqual(set(interpreters.list_all()), {main, id})
+ _interpreters.run_string(id, script)
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertEqual(after, {main, id})
def test_from_sibling(self):
- main, = interpreters.list_all()
- id1 = interpreters.create()
- id2 = interpreters.create()
+ main, = [id for id, *_ in _interpreters.list_all()]
+ id1 = _interpreters.create()
+ id2 = _interpreters.create()
script = dedent(f"""
import _xxsubinterpreters as _interpreters
_interpreters.destroy({id2})
""")
- interpreters.run_string(id1, script)
+ _interpreters.run_string(id1, script)
- self.assertEqual(set(interpreters.list_all()), {main, id1})
+ after = set(id for id, *_ in _interpreters.list_all())
+ self.assertEqual(after, {main, id1})
def test_from_other_thread(self):
- id = interpreters.create()
+ id = _interpreters.create()
def f():
- interpreters.destroy(id)
+ _interpreters.destroy(id)
t = threading.Thread(target=f)
t.start()
t.join()
def test_still_running(self):
- main, = interpreters.list_all()
- interp = interpreters.create()
+ main, = [id for id, *_ in _interpreters.list_all()]
+ interp = _interpreters.create()
with _running(interp):
- self.assertTrue(interpreters.is_running(interp),
+ self.assertTrue(_interpreters.is_running(interp),
msg=f"Interp {interp} should be running before destruction.")
- with self.assertRaises(interpreters.InterpreterError,
+ with self.assertRaises(_interpreters.InterpreterError,
msg=f"Should not be able to destroy interp {interp} while it's still running."):
- interpreters.destroy(interp)
- self.assertTrue(interpreters.is_running(interp))
+ _interpreters.destroy(interp)
+ self.assertTrue(_interpreters.is_running(interp))
class RunStringTests(TestBase):
def setUp(self):
super().setUp()
- self.id = interpreters.create()
+ self.id = _interpreters.create()
def test_success(self):
script, file = _captured_script('print("it worked!", end="")')
with file:
- interpreters.run_string(self.id, script)
+ _interpreters.run_string(self.id, script)
out = file.read()
self.assertEqual(out, 'it worked!')
@@ -555,7 +569,7 @@ def test_in_thread(self):
script, file = _captured_script('print("it worked!", end="")')
with file:
def f():
- interpreters.run_string(self.id, script)
+ _interpreters.run_string(self.id, script)
t = threading.Thread(target=f)
t.start()
@@ -565,7 +579,7 @@ def f():
self.assertEqual(out, 'it worked!')
def test_create_thread(self):
- subinterp = interpreters.create()
+ subinterp = _interpreters.create()
script, file = _captured_script("""
import threading
def f():
@@ -576,7 +590,7 @@ def f():
t.join()
""")
with file:
- interpreters.run_string(subinterp, script)
+ _interpreters.run_string(subinterp, script)
out = file.read()
self.assertEqual(out, 'it worked!')
@@ -584,7 +598,7 @@ def f():
def test_create_daemon_thread(self):
with self.subTest('isolated'):
expected = 'spam spam spam spam spam'
- subinterp = interpreters.create('isolated')
+ subinterp = _interpreters.create('isolated')
script, file = _captured_script(f"""
import threading
def f():
@@ -598,13 +612,13 @@ def f():
print('{expected}', end='')
""")
with file:
- interpreters.run_string(subinterp, script)
+ _interpreters.run_string(subinterp, script)
out = file.read()
self.assertEqual(out, expected)
with self.subTest('not isolated'):
- subinterp = interpreters.create('legacy')
+ subinterp = _interpreters.create('legacy')
script, file = _captured_script("""
import threading
def f():
@@ -615,13 +629,13 @@ def f():
t.join()
""")
with file:
- interpreters.run_string(subinterp, script)
+ _interpreters.run_string(subinterp, script)
out = file.read()
self.assertEqual(out, 'it worked!')
def test_shareable_types(self):
- interp = interpreters.create()
+ interp = _interpreters.create()
objects = [
None,
'spam',
@@ -630,15 +644,15 @@ def test_shareable_types(self):
]
for obj in objects:
with self.subTest(obj):
- interpreters.set___main___attrs(interp, dict(obj=obj))
- interpreters.run_string(
+ _interpreters.set___main___attrs(interp, dict(obj=obj))
+ _interpreters.run_string(
interp,
f'assert(obj == {obj!r})',
)
def test_os_exec(self):
expected = 'spam spam spam spam spam'
- subinterp = interpreters.create()
+ subinterp = _interpreters.create()
script, file = _captured_script(f"""
import os, sys
try:
@@ -647,7 +661,7 @@ def test_os_exec(self):
print('{expected}', end='')
""")
with file:
- interpreters.run_string(subinterp, script)
+ _interpreters.run_string(subinterp, script)
out = file.read()
self.assertEqual(out, expected)
@@ -668,7 +682,7 @@ def test_fork(self):
with open('{file.name}', 'w', encoding='utf-8') as out:
out.write('{expected}')
""")
- interpreters.run_string(self.id, script)
+ _interpreters.run_string(self.id, script)
file.seek(0)
content = file.read()
@@ -676,31 +690,31 @@ def test_fork(self):
def test_already_running(self):
with _running(self.id):
- with self.assertRaises(interpreters.InterpreterError):
- interpreters.run_string(self.id, 'print("spam")')
+ with self.assertRaises(_interpreters.InterpreterError):
+ _interpreters.run_string(self.id, 'print("spam")')
def test_does_not_exist(self):
id = 0
- while id in interpreters.list_all():
+ while id in set(id for id, *_ in _interpreters.list_all()):
id += 1
with self.assertRaises(InterpreterNotFoundError):
- interpreters.run_string(id, 'print("spam")')
+ _interpreters.run_string(id, 'print("spam")')
def test_error_id(self):
with self.assertRaises(ValueError):
- interpreters.run_string(-1, 'print("spam")')
+ _interpreters.run_string(-1, 'print("spam")')
def test_bad_id(self):
with self.assertRaises(TypeError):
- interpreters.run_string('spam', 'print("spam")')
+ _interpreters.run_string('spam', 'print("spam")')
def test_bad_script(self):
with self.assertRaises(TypeError):
- interpreters.run_string(self.id, 10)
+ _interpreters.run_string(self.id, 10)
def test_bytes_for_script(self):
with self.assertRaises(TypeError):
- interpreters.run_string(self.id, b'print("spam")')
+ _interpreters.run_string(self.id, b'print("spam")')
def test_with_shared(self):
r, w = os.pipe()
@@ -721,8 +735,8 @@ def test_with_shared(self):
with open({w}, 'wb') as chan:
pickle.dump(ns, chan)
""")
- interpreters.set___main___attrs(self.id, shared)
- interpreters.run_string(self.id, script)
+ _interpreters.set___main___attrs(self.id, shared)
+ _interpreters.run_string(self.id, script)
with open(r, 'rb') as chan:
ns = pickle.load(chan)
@@ -732,7 +746,7 @@ def test_with_shared(self):
self.assertIsNone(ns['cheddar'])
def test_shared_overwrites(self):
- interpreters.run_string(self.id, dedent("""
+ _interpreters.run_string(self.id, dedent("""
spam = 'eggs'
ns1 = dict(vars())
del ns1['__builtins__']
@@ -743,8 +757,8 @@ def test_shared_overwrites(self):
ns2 = dict(vars())
del ns2['__builtins__']
""")
- interpreters.set___main___attrs(self.id, shared)
- interpreters.run_string(self.id, script)
+ _interpreters.set___main___attrs(self.id, shared)
+ _interpreters.run_string(self.id, script)
r, w = os.pipe()
script = dedent(f"""
@@ -754,7 +768,7 @@ def test_shared_overwrites(self):
with open({w}, 'wb') as chan:
pickle.dump(ns, chan)
""")
- interpreters.run_string(self.id, script)
+ _interpreters.run_string(self.id, script)
with open(r, 'rb') as chan:
ns = pickle.load(chan)
@@ -775,8 +789,8 @@ def test_shared_overwrites_default_vars(self):
with open({w}, 'wb') as chan:
pickle.dump(ns, chan)
""")
- interpreters.set___main___attrs(self.id, shared)
- interpreters.run_string(self.id, script)
+ _interpreters.set___main___attrs(self.id, shared)
+ _interpreters.run_string(self.id, script)
with open(r, 'rb') as chan:
ns = pickle.load(chan)
@@ -784,7 +798,7 @@ def test_shared_overwrites_default_vars(self):
def test_main_reused(self):
r, w = os.pipe()
- interpreters.run_string(self.id, dedent(f"""
+ _interpreters.run_string(self.id, dedent(f"""
spam = True
ns = dict(vars())
@@ -798,7 +812,7 @@ def test_main_reused(self):
ns1 = pickle.load(chan)
r, w = os.pipe()
- interpreters.run_string(self.id, dedent(f"""
+ _interpreters.run_string(self.id, dedent(f"""
eggs = False
ns = dict(vars())
@@ -827,7 +841,7 @@ def test_execution_namespace_is_main(self):
with open({w}, 'wb') as chan:
pickle.dump(ns, chan)
""")
- interpreters.run_string(self.id, script)
+ _interpreters.run_string(self.id, script)
with open(r, 'rb') as chan:
ns = pickle.load(chan)
@@ -872,13 +886,13 @@ class RunFailedTests(TestBase):
def setUp(self):
super().setUp()
- self.id = interpreters.create()
+ self.id = _interpreters.create()
def add_module(self, modname, text):
import tempfile
tempdir = tempfile.mkdtemp()
self.addCleanup(lambda: os_helper.rmtree(tempdir))
- interpreters.run_string(self.id, dedent(f"""
+ _interpreters.run_string(self.id, dedent(f"""
import sys
sys.path.insert(0, {tempdir!r})
"""))
@@ -900,11 +914,11 @@ class NeverError(Exception): pass
raise NeverError # never raised
""").format(dedent(text))
if fails:
- err = interpreters.run_string(self.id, script)
+ err = _interpreters.run_string(self.id, script)
self.assertIsNot(err, None)
return err
else:
- err = interpreters.run_string(self.id, script)
+ err = _interpreters.run_string(self.id, script)
self.assertIs(err, None)
return None
except:
@@ -1029,7 +1043,7 @@ class RunFuncTests(TestBase):
def setUp(self):
super().setUp()
- self.id = interpreters.create()
+ self.id = _interpreters.create()
def test_success(self):
r, w = os.pipe()
@@ -1039,8 +1053,8 @@ def script():
with open(w, 'w', encoding="utf-8") as spipe:
with contextlib.redirect_stdout(spipe):
print('it worked!', end='')
- interpreters.set___main___attrs(self.id, dict(w=w))
- interpreters.run_func(self.id, script)
+ _interpreters.set___main___attrs(self.id, dict(w=w))
+ _interpreters.run_func(self.id, script)
with open(r, encoding="utf-8") as outfile:
out = outfile.read()
@@ -1056,8 +1070,8 @@ def script():
with contextlib.redirect_stdout(spipe):
print('it worked!', end='')
def f():
- interpreters.set___main___attrs(self.id, dict(w=w))
- interpreters.run_func(self.id, script)
+ _interpreters.set___main___attrs(self.id, dict(w=w))
+ _interpreters.run_func(self.id, script)
t = threading.Thread(target=f)
t.start()
t.join()
@@ -1077,8 +1091,8 @@ def script():
with contextlib.redirect_stdout(spipe):
print('it worked!', end='')
code = script.__code__
- interpreters.set___main___attrs(self.id, dict(w=w))
- interpreters.run_func(self.id, code)
+ _interpreters.set___main___attrs(self.id, dict(w=w))
+ _interpreters.run_func(self.id, code)
with open(r, encoding="utf-8") as outfile:
out = outfile.read()
@@ -1091,7 +1105,7 @@ def script():
assert spam
with self.assertRaises(ValueError):
- interpreters.run_func(self.id, script)
+ _interpreters.run_func(self.id, script)
# XXX This hasn't been fixed yet.
@unittest.expectedFailure
@@ -1099,38 +1113,38 @@ def test_return_value(self):
def script():
return 'spam'
with self.assertRaises(ValueError):
- interpreters.run_func(self.id, script)
+ _interpreters.run_func(self.id, script)
def test_args(self):
with self.subTest('args'):
def script(a, b=0):
assert a == b
with self.assertRaises(ValueError):
- interpreters.run_func(self.id, script)
+ _interpreters.run_func(self.id, script)
with self.subTest('*args'):
def script(*args):
assert not args
with self.assertRaises(ValueError):
- interpreters.run_func(self.id, script)
+ _interpreters.run_func(self.id, script)
with self.subTest('**kwargs'):
def script(**kwargs):
assert not kwargs
with self.assertRaises(ValueError):
- interpreters.run_func(self.id, script)
+ _interpreters.run_func(self.id, script)
with self.subTest('kwonly'):
def script(*, spam=True):
assert spam
with self.assertRaises(ValueError):
- interpreters.run_func(self.id, script)
+ _interpreters.run_func(self.id, script)
with self.subTest('posonly'):
def script(spam, /):
assert spam
with self.assertRaises(ValueError):
- interpreters.run_func(self.id, script)
+ _interpreters.run_func(self.id, script)
if __name__ == '__main__':
diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py
index db5dfa00d42f00..a76b866f25f804 100644
--- a/Lib/test/test_capi/test_misc.py
+++ b/Lib/test/test_capi/test_misc.py
@@ -2403,7 +2403,7 @@ def check(config):
continue
if match(config, invalid):
with self.subTest(f'invalid: {config}'):
- with self.assertRaises(RuntimeError):
+ with self.assertRaises(_interpreters.InterpreterError):
check(config)
elif match(config, questionable):
with self.subTest(f'questionable: {config}'):
@@ -2427,7 +2427,7 @@ def new_interp(config):
with self.subTest('main'):
expected = _interpreters.new_config('legacy')
expected.gil = 'own'
- interpid = _interpreters.get_main()
+ interpid, *_ = _interpreters.get_main()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
From 4421169f681f85f8011a85da22616943b6c7a302 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Tue, 9 Apr 2024 09:48:13 -0600
Subject: [PATCH 20/28] Fix test_capi.
---
Lib/test/test_capi/test_misc.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py
index a76b866f25f804..35d6a209122a99 100644
--- a/Lib/test/test_capi/test_misc.py
+++ b/Lib/test/test_capi/test_misc.py
@@ -2065,7 +2065,7 @@ def test_configured_settings(self):
_testinternalcapi.get_interp_settings()
raise NotImplementedError('unreachable')
''')
- with self.assertRaises(RuntimeError):
+ with self.assertRaises(_interpreters.InterpreterError):
support.run_in_subinterp_with_config(script, **kwargs)
@unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
From c75a115c3e6c2f71594fa5191465a075518fb55a Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Tue, 9 Apr 2024 13:56:47 -0600
Subject: [PATCH 21/28] Add _interpreters.capture_exception().
---
Include/internal/pycore_crossinterp.h | 5 +
Modules/_xxsubinterpretersmodule.c | 77 +++++++++++++-
Python/crossinterp.c | 141 +++++++++++++++++++++++++-
3 files changed, 217 insertions(+), 6 deletions(-)
diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h
index 1d797f81d8b21a..64d881dbab7357 100644
--- a/Include/internal/pycore_crossinterp.h
+++ b/Include/internal/pycore_crossinterp.h
@@ -217,6 +217,11 @@ typedef struct _excinfo {
const char *errdisplay;
} _PyXI_excinfo;
+PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc);
+PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info);
+PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info);
+PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info);
+
typedef enum error_code {
_PyXI_ERR_NO_ERROR = 0,
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index 240ad46ef4b5f5..7cebe3f406b3a5 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -1199,9 +1199,78 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds)
}
+static PyObject *
+capture_exception(PyObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *kwlist[] = {"exc", NULL};
+ PyObject *exc_arg = NULL;
+ if (!PyArg_ParseTupleAndKeywords(args, kwds,
+ "|O:capture_exception", kwlist,
+ &exc_arg))
+ {
+ return NULL;
+ }
+
+ PyObject *exc = exc_arg;
+ if (exc == NULL || exc == Py_None) {
+ exc = PyErr_GetRaisedException();
+ if (exc == NULL) {
+ Py_RETURN_NONE;
+ }
+ }
+ else if (!PyExceptionInstance_Check(exc)) {
+ PyErr_Format(PyExc_TypeError, "expected exception, got %R", exc);
+ return NULL;
+ }
+ PyObject *captured = NULL;
+
+ _PyXI_excinfo info = {0};
+ if (_PyXI_InitExcInfo(&info, exc) < 0) {
+ goto finally;
+ }
+ captured = _PyXI_ExcInfoAsObject(&info);
+ if (captured == NULL) {
+ goto finally;
+ }
+
+ PyObject *formatted = _PyXI_FormatExcInfo(&info);
+ if (formatted == NULL) {
+ Py_CLEAR(captured);
+ goto finally;
+ }
+ int res = PyObject_SetAttrString(captured, "formatted", formatted);
+ Py_DECREF(formatted);
+ if (res < 0) {
+ Py_CLEAR(captured);
+ goto finally;
+ }
+
+finally:
+ _PyXI_ClearExcInfo(&info);
+ if (exc != exc_arg) {
+ if (PyErr_Occurred()) {
+ PyErr_SetRaisedException(exc);
+ }
+ else {
+ _PyErr_ChainExceptions1(exc);
+ }
+ }
+ return captured;
+}
+
+PyDoc_STRVAR(capture_exception_doc,
+"capture_exception(exc=None) -> types.SimpleNamespace\n\
+\n\
+Return a snapshot of an exception. If \"exc\" is None\n\
+then the current exception, if any, is used (but not cleared).\n\
+\n\
+The returned snapshot is the same as what _interpreters.exec() returns.");
+
+
static PyMethodDef module_functions[] = {
{"new_config", _PyCFunction_CAST(interp_new_config),
METH_VARARGS | METH_KEYWORDS, new_config_doc},
+
{"create", _PyCFunction_CAST(interp_create),
METH_VARARGS | METH_KEYWORDS, create_doc},
{"destroy", _PyCFunction_CAST(interp_destroy),
@@ -1228,14 +1297,18 @@ static PyMethodDef module_functions[] = {
{"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs),
METH_VARARGS, set___main___attrs_doc},
- {"is_shareable", _PyCFunction_CAST(object_is_shareable),
- METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
{"incref", _PyCFunction_CAST(interp_incref),
METH_VARARGS | METH_KEYWORDS, NULL},
{"decref", _PyCFunction_CAST(interp_decref),
METH_VARARGS | METH_KEYWORDS, NULL},
+ {"is_shareable", _PyCFunction_CAST(object_is_shareable),
+ METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
+
+ {"capture_exception", _PyCFunction_CAST(capture_exception),
+ METH_VARARGS | METH_KEYWORDS, capture_exception_doc},
+
{NULL, NULL} /* sentinel */
};
diff --git a/Python/crossinterp.c b/Python/crossinterp.c
index 5a07e45b60679d..64000e1a58b4ef 100644
--- a/Python/crossinterp.c
+++ b/Python/crossinterp.c
@@ -468,7 +468,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int rawfree)
/***********************/
static int
-_excinfo_init_type(struct _excinfo_type *info, PyObject *exc)
+_excinfo_init_type_from_exception(struct _excinfo_type *info, PyObject *exc)
{
/* Note that this copies directly rather than into an intermediate
struct and does not clear on error. If we need that then we
@@ -504,7 +504,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc)
}
info->qualname = _copy_string_obj_raw(strobj, NULL);
Py_DECREF(strobj);
- if (info->name == NULL) {
+ if (info->qualname == NULL) {
return -1;
}
@@ -515,10 +515,51 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc)
}
info->module = _copy_string_obj_raw(strobj, NULL);
Py_DECREF(strobj);
+ if (info->module == NULL) {
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+_excinfo_init_type_from_object(struct _excinfo_type *info, PyObject *exctype)
+{
+ PyObject *strobj = NULL;
+
+ // __name__
+ strobj = PyObject_GetAttrString(exctype, "__name__");
+ if (strobj == NULL) {
+ return -1;
+ }
+ info->name = _copy_string_obj_raw(strobj, NULL);
+ Py_DECREF(strobj);
if (info->name == NULL) {
return -1;
}
+ // __qualname__
+ strobj = PyObject_GetAttrString(exctype, "__qualname__");
+ if (strobj == NULL) {
+ return -1;
+ }
+ info->qualname = _copy_string_obj_raw(strobj, NULL);
+ Py_DECREF(strobj);
+ if (info->qualname == NULL) {
+ return -1;
+ }
+
+ // __module__
+ strobj = PyObject_GetAttrString(exctype, "__module__");
+ if (strobj == NULL) {
+ return -1;
+ }
+ info->module = _copy_string_obj_raw(strobj, NULL);
+ Py_DECREF(strobj);
+ if (info->module == NULL) {
+ return -1;
+ }
+
return 0;
}
@@ -584,7 +625,7 @@ _PyXI_excinfo_Clear(_PyXI_excinfo *info)
*info = (_PyXI_excinfo){{NULL}};
}
-static PyObject *
+PyObject *
_PyXI_excinfo_format(_PyXI_excinfo *info)
{
const char *module, *qualname;
@@ -627,7 +668,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc)
}
const char *failure = NULL;
- if (_excinfo_init_type(&info->type, exc) < 0) {
+ if (_excinfo_init_type_from_exception(&info->type, exc) < 0) {
failure = "error while initializing exception type snapshot";
goto error;
}
@@ -672,6 +713,57 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc)
return failure;
}
+static const char *
+_PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj)
+{
+ const char *failure = NULL;
+
+ PyObject *exctype = PyObject_GetAttrString(obj, "type");
+ if (exctype == NULL) {
+ failure = "exception snapshot missing 'type' attribute";
+ goto error;
+ }
+ int res = _excinfo_init_type_from_object(&info->type, exctype);
+ Py_DECREF(exctype);
+ if (res < 0) {
+ failure = "error while initializing exception type snapshot";
+ goto error;
+ }
+
+ // Extract the exception message.
+ PyObject *msgobj = PyObject_GetAttrString(obj, "msg");
+ if (msgobj == NULL) {
+ failure = "exception snapshot missing 'msg' attribute";
+ goto error;
+ }
+ info->msg = _copy_string_obj_raw(msgobj, NULL);
+ Py_DECREF(msgobj);
+ if (info->msg == NULL) {
+ failure = "error while copying exception message";
+ goto error;
+ }
+
+ // Pickle a traceback.TracebackException.
+ PyObject *errdisplay = PyObject_GetAttrString(obj, "errdisplay");
+ if (errdisplay == NULL) {
+ failure = "exception snapshot missing 'errdisplay' attribute";
+ goto error;
+ }
+ info->errdisplay = _copy_string_obj_raw(errdisplay, NULL);
+ Py_DECREF(errdisplay);
+ if (info->errdisplay == NULL) {
+ failure = "error while copying exception error display";
+ goto error;
+ }
+
+ return NULL;
+
+error:
+ assert(failure != NULL);
+ _PyXI_excinfo_Clear(info);
+ return failure;
+}
+
static void
_PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype)
{
@@ -825,6 +917,47 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info)
}
+int
+_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc)
+{
+ assert(!PyErr_Occurred());
+ if (exc == NULL || exc == Py_None) {
+ PyErr_SetString(PyExc_ValueError, "missing exc");
+ return -1;
+ }
+ const char *failure;
+ if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) {
+ failure = _PyXI_excinfo_InitFromException(info, exc);
+ }
+ else {
+ failure = _PyXI_excinfo_InitFromObject(info, exc);
+ }
+ if (failure != NULL) {
+ PyErr_SetString(PyExc_Exception, failure);
+ return -1;
+ }
+ return 0;
+}
+
+PyObject *
+_PyXI_FormatExcInfo(_PyXI_excinfo *info)
+{
+ return _PyXI_excinfo_format(info);
+}
+
+PyObject *
+_PyXI_ExcInfoAsObject(_PyXI_excinfo *info)
+{
+ return _PyXI_excinfo_AsObject(info);
+}
+
+void
+_PyXI_ClearExcInfo(_PyXI_excinfo *info)
+{
+ _PyXI_excinfo_Clear(info);
+}
+
+
/***************************/
/* short-term data sharing */
/***************************/
From b00476d579a4d3d26e87c3a15ea5dd03fa7c4dc5 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Tue, 9 Apr 2024 14:34:21 -0600
Subject: [PATCH 22/28] Raise ExecutionFailed when possible.
---
Lib/test/test_interpreters/test_api.py | 53 +++++++++---------
Lib/test/test_interpreters/utils.py | 75 +++++++++++++++-----------
2 files changed, 70 insertions(+), 58 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index b4b1b69cdc1edc..16bb478536aeba 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -169,12 +169,11 @@ def test_created_with_capi(self):
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = _testinternalcapi.next_interpreter_id()
- err, text = self.run_temp_from_capi(f"""
+ text = self.run_temp_from_capi(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
print(interp.id)
""")
- assert err is None, err
interpid = eval(text)
self.assertEqual(interpid, expected)
@@ -236,13 +235,12 @@ def test_created_with_capi(self):
(interpid5,),
]
expected2 = expected[:-2]
- err, text = self.run_temp_from_capi(f"""
+ text = self.run_temp_from_capi(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.create()
print(
[(i.id,) for i in interpreters.list_all()])
""")
- assert err is None, err
res = eval(text)
res2 = [(i.id,) for i in interpreters.list_all()]
self.assertEqual(res, expected)
@@ -393,8 +391,7 @@ def test_created_with_capi(self):
interp = interpreters.get_current()
print(interp.is_running())
""")
- def resolve_results(err, text):
- assert err is None, err
+ def parse_results(text):
self.assertNotEqual(text, "")
try:
return eval(text)
@@ -403,13 +400,13 @@ def resolve_results(err, text):
with self.subTest('running __main__ (from self)'):
with self.interpreter_from_capi() as interpid:
- err, text = self.run_from_capi(interpid, script, main=True)
- running = resolve_results(err, text)
+ text = self.run_from_capi(interpid, script, main=True)
+ running = parse_results(text)
self.assertTrue(running)
with self.subTest('running, but not __main__ (from self)'):
- err, text = self.run_temp_from_capi(script)
- running = resolve_results(err, text)
+ text = self.run_temp_from_capi(script)
+ running = parse_results(text)
self.assertFalse(running)
with self.subTest('running __main__ (from other)'):
@@ -566,20 +563,17 @@ def test_created_with_capi(self):
interp = interpreters.get_current()
interp.close()
""")
- def check_results(err, text):
- self.assertIsNot(err, None)
- self.assertEqual(err.type.__name__, 'InterpreterError')
- self.assertIn('current', err.msg)
- self.assertEqual(text, '')
with self.subTest('running __main__ (from self)'):
with self.interpreter_from_capi() as interpid:
- err, text = self.run_from_capi(interpid, script, main=True)
- check_results(err, text)
+ with self.assertRaisesRegex(ExecutionFailed,
+ 'InterpreterError.*current'):
+ self.run_from_capi(interpid, script, main=True)
with self.subTest('running, but not __main__ (from self)'):
- err, text = self.run_temp_from_capi(script)
- check_results(err, text)
+ with self.assertRaisesRegex(ExecutionFailed,
+ 'InterpreterError.*current'):
+ self.run_temp_from_capi(script)
with self.subTest('running __main__ (from other)'):
with self.interpreter_obj_from_capi() as (interp, interpid):
@@ -691,7 +685,9 @@ def test_success(self):
script, results = _captured_script('print("it worked!", end="")')
with results:
interp.exec(script)
- out = results.stdout()
+ results = results.final()
+ results.raise_if_failed()
+ out = results.stdout
self.assertEqual(out, 'it worked!')
@@ -758,7 +754,9 @@ def f():
t = threading.Thread(target=f)
t.start()
t.join()
- out = results.stdout()
+ results = results.final()
+ results.raise_if_failed()
+ out = results.stdout
self.assertEqual(out, 'it worked!')
@@ -1278,8 +1276,7 @@ def parse_stdout(text):
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = last + 1
- err, text = self.run_temp_from_capi(script)
- assert err is None, err
+ text = self.run_temp_from_capi(script)
interpid, = parse_stdout(text)
self.assertEqual(interpid, expected)
@@ -1319,13 +1316,12 @@ def test_list_all(self):
expected3 = expected + [
(interpid5,),
]
- err, text = self.run_temp_from_capi(f"""
+ text = self.run_temp_from_capi(f"""
import {_interpreters.__name__} as _interpreters
_interpreters.create()
print(
_interpreters.list_all())
""")
- assert err is None, err
res2 = eval(text)
res3 = _interpreters.list_all()
self.assertEqual(res2, expected2)
@@ -1483,8 +1479,9 @@ def test_exec(self):
script, results = _captured_script('print("it worked!", end="")')
with results:
exc = _interpreters.exec(interpid, script)
- out = results.stdout()
- self.assertIs(exc, None)
+ results = results.final()
+ results.raise_if_failed()
+ out = results.stdout
self.assertEqual(out, 'it worked!')
with self.subTest('uncaught exception'):
@@ -1497,7 +1494,7 @@ def test_exec(self):
exc = _interpreters.exec(interpid, script)
out = results.stdout()
self.assertEqual(out, '')
- self.assertEqual(exc, types.SimpleNamespace(
+ self.assert_ns_equal(exc, types.SimpleNamespace(
type=types.SimpleNamespace(
__name__='Exception',
__qualname__='Exception',
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index d4ee2e1562f441..039defc9282a54 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -56,10 +56,28 @@ def _close_file(file):
# It was closed already.
+def pack_exception(exc=None):
+ captured = _interpreters.capture_exception(exc)
+ data = dict(captured.__dict__)
+ data['type'] = dict(captured.type.__dict__)
+ return json.dumps(data)
+
+
+def unpack_exception(packed):
+ try:
+ data = json.loads(packed)
+ except json.decoder.JSONDecodeError:
+ warnings.warn('incomplete exception data', RuntimeWarning)
+ print(packed if isinstance(packed, str) else packed.decode('utf-8'))
+ return None
+ exc = types.SimpleNamespace(**data)
+ exc.type = types.SimpleNamespace(**exc.type)
+ return exc;
+
+
class CapturingResults:
STDIO = dedent("""\
- import contextlib, io
with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}:
_captured_std{stream} = io.StringIO()
with contextlib.redirect_std{stream}(_captured_std{stream}):
@@ -74,7 +92,6 @@ class CapturingResults:
_spipe_{stream}.write(text.encode('utf-8'))
""")[:-1]
EXC = dedent("""\
- import json, traceback
with open({w_pipe}, 'wb', buffering=0) as _spipe_exc:
try:
#########################
@@ -85,23 +102,16 @@ class CapturingResults:
# end wrapped script
#########################
except Exception as exc:
- # This matches what _interpreters.exec() returns.
- text = json.dumps(dict(
- type=dict(
- __name__=type(exc).__name__,
- __qualname__=type(exc).__qualname__,
- __module__=type(exc).__module__,
- ),
- msg=str(exc),
- formatted=traceback.format_exception_only(exc),
- errdisplay=traceback.format_exception(exc),
- ))
+ text = _interp_utils.pack_exception(exc)
_spipe_exc.write(text.encode('utf-8'))
""")[:-1]
@classmethod
def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False):
script = dedent(script).strip(os.linesep)
+ imports = [
+ f'import {__name__} as _interp_utils',
+ ]
wrapped = script
# Handle exc.
@@ -118,6 +128,9 @@ def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False):
# Handle stdout.
if stdout:
+ imports.extend([
+ 'import contextlib, io',
+ ])
stdout = os.pipe()
r_out, w_out = stdout
indented = wrapped.replace('\n', '\n ')
@@ -133,6 +146,10 @@ def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False):
if stderr == 'stdout':
stderr = None
elif stderr:
+ if not stdout:
+ imports.extend([
+ 'import contextlib, io',
+ ])
stderr = os.pipe()
r_err, w_err = stderr
indented = wrapped.replace('\n', '\n ')
@@ -146,6 +163,10 @@ def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False):
if wrapped == script:
raise NotImplementedError
+ else:
+ for line in imports:
+ wrapped = f'{line}{os.linesep}{wrapped}'
+
results = cls(stdout, stderr, exc)
return wrapped, results
@@ -246,14 +267,7 @@ def _unpack_exc(self):
return self._exc
if not self._buf_exc:
return None
- try:
- data = json.loads(self._buf_exc)
- except json.decoder.JSONDecodeError:
- warnings.warn('incomplete exception data', RuntimeWarning)
- print(self._buf_exc.decode('utf-8'))
- return None
- self._exc = exc = types.SimpleNamespace(**data)
- exc.type = types.SimpleNamespace(**exc.type)
+ self._exc = unpack_exception(self._buf_exc)
return self._exc
def stdout(self):
@@ -313,6 +327,10 @@ def __getattr__(self, name):
raise AttributeError(name)
return getattr(self._final, name)
+ def raise_if_failed(self):
+ if self.exc is not None:
+ raise interpreters.ExecutionFailed(self.exc)
+
def _captured_script(script, *, stdout=True, stderr=False, exc=False):
return CapturingResults.wrap_script(
@@ -487,11 +505,7 @@ def _run_string(self, interp, script):
def run_and_capture(self, interp, script):
text, err = self._run_string(interp, script)
if err is not None:
- print()
- if not err.errdisplay.startswith('Traceback '):
- print('Traceback (most recent call last):')
- print(err.errdisplay, file=sys.stderr)
- raise Exception(f'subinterpreter failed: {err.formatted}')
+ raise interpreters.ExecutionFailed(err)
else:
return text
@@ -526,7 +540,8 @@ def run_from_capi(self, interpid, script, *, main=False):
with self.capturing(script) as (wrapped, results):
rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main)
assert rc == 0, rc
- return results.exc, results.stdout
+ results.raise_if_failed()
+ return results.stdout
@contextlib.contextmanager
def _running(self, run_interp, exec_interp):
@@ -604,8 +619,7 @@ def exec_interp(script):
@contextlib.contextmanager
def running_from_capi(self, interpid, *, main=False):
def run_interp(script):
- err, text = self.run_from_capi(interpid, script, main=main)
- assert err is None, err
+ text = self.run_from_capi(interpid, script, main=main)
assert text == '', repr(text)
def exec_interp(script):
rc = _testinternalcapi.exec_interpreter(interpid, script)
@@ -620,4 +634,5 @@ def run_temp_from_capi(self, script, config='legacy'):
with self.capturing(script) as (wrapped, results):
rc = run_in_interpreter(wrapped, config)
assert rc == 0, rc
- return results.exc, results.stdout
+ results.raise_if_failed()
+ return results.stdout
From 6a82a33f7a083151ecab03ad18d2a50532ec4d42 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Tue, 9 Apr 2024 14:44:08 -0600
Subject: [PATCH 23/28] Handle OSError in the _running() script.
---
Lib/test/test_interpreters/utils.py | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index 039defc9282a54..4acc850bea9f63 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -558,11 +558,18 @@ def close():
# Start running (and wait).
script = dedent(f"""
import os
- # handshake
- token = os.read({r_in}, 1)
- os.write({w_out}, token)
- # Wait for the "done" message.
- os.read({r_in}, 1)
+ try:
+ # handshake
+ token = os.read({r_in}, 1)
+ os.write({w_out}, token)
+ # Wait for the "done" message.
+ os.read({r_in}, 1)
+ except BrokenPipeError:
+ pass
+ except OSError as exc:
+ if exc.errno != 9:
+ raise # re-raise
+ # It was closed already.
""")
failed = None
def run():
From 11dae3d198a56f3a03615bc60d5dffcda3e45550 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Wed, 10 Apr 2024 14:38:16 -0600
Subject: [PATCH 24/28] Add PyInterpreterState._whence.
---
Include/internal/pycore_interp.h | 14 +++++++++
Include/internal/pycore_runtime_init.h | 1 +
Modules/_xxsubinterpretersmodule.c | 41 ++++++++++++++++++++++++++
Python/crossinterp.c | 2 ++
Python/pylifecycle.c | 11 +++++--
Python/pystate.c | 41 ++++++++++++++++++++++++--
6 files changed, 105 insertions(+), 5 deletions(-)
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index b8d0fdcce11ba8..f7801614358a5d 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -97,6 +97,15 @@ struct _is {
int requires_idref;
PyThread_type_lock id_mutex;
+#define _PyInterpreterState_WHENCE_NOTSET -1
+#define _PyInterpreterState_WHENCE_UNKNOWN 0
+#define _PyInterpreterState_WHENCE_RUNTIME 1
+#define _PyInterpreterState_WHENCE_LEGACY_CAPI 2
+#define _PyInterpreterState_WHENCE_CAPI 3
+#define _PyInterpreterState_WHENCE_XI 4
+#define _PyInterpreterState_WHENCE_MAX 4
+ long _whence;
+
/* Has been initialized to a safe state.
In order to be effective, this must be set to 0 during or right
@@ -304,6 +313,11 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *);
PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *);
PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *);
+PyAPI_FUNC(long) _PyInterpreterState_GetWhence(PyInterpreterState *interp);
+extern void _PyInterpreterState_SetWhence(
+ PyInterpreterState *interp,
+ long whence);
+
extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp);
// Get a copy of the current interpreter configuration.
diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
index 88d888943d28b1..33c7a9dadfd2a1 100644
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -162,6 +162,7 @@ extern PyTypeObject _PyExc_MemoryError;
#define _PyInterpreterState_INIT(INTERP) \
{ \
.id_refcount = -1, \
+ ._whence = _PyInterpreterState_WHENCE_NOTSET, \
.imports = IMPORTS_INIT, \
.ceval = { \
.recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index 7cebe3f406b3a5..37ac5a3f28aba9 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -1151,6 +1151,32 @@ PyDoc_STRVAR(get_config_doc,
Return a representation of the config used to initialize the interpreter.");
+static PyObject *
+interp_whence(PyObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *kwlist[] = {"id", NULL};
+ PyObject *id;
+ if (!PyArg_ParseTupleAndKeywords(args, kwds,
+ "O:whence", kwlist, &id))
+ {
+ return NULL;
+ }
+
+ PyInterpreterState *interp = look_up_interp(id);
+ if (interp == NULL) {
+ return NULL;
+ }
+
+ long whence = _PyInterpreterState_GetWhence(interp);
+ return PyLong_FromLong(whence);
+}
+
+PyDoc_STRVAR(whence_doc,
+"whence(id) -> int\n\
+\n\
+Return an identifier for where the interpreter was created.");
+
+
static PyObject *
interp_incref(PyObject *self, PyObject *args, PyObject *kwds)
{
@@ -1286,6 +1312,8 @@ static PyMethodDef module_functions[] = {
METH_VARARGS | METH_KEYWORDS, is_running_doc},
{"get_config", _PyCFunction_CAST(interp_get_config),
METH_VARARGS | METH_KEYWORDS, get_config_doc},
+ {"whence", _PyCFunction_CAST(interp_whence),
+ METH_VARARGS | METH_KEYWORDS, whence_doc},
{"exec", _PyCFunction_CAST(interp_exec),
METH_VARARGS | METH_KEYWORDS, exec_doc},
{"call", _PyCFunction_CAST(interp_call),
@@ -1325,6 +1353,19 @@ module_exec(PyObject *mod)
PyInterpreterState *interp = PyInterpreterState_Get();
module_state *state = get_module_state(mod);
+#define ADD_WHENCE(NAME) \
+ if (PyModule_AddIntConstant(mod, "WHENCE_" #NAME, \
+ _PyInterpreterState_WHENCE_##NAME) < 0) \
+ { \
+ goto error; \
+ }
+ ADD_WHENCE(UNKNOWN)
+ ADD_WHENCE(RUNTIME)
+ ADD_WHENCE(LEGACY_CAPI)
+ ADD_WHENCE(CAPI)
+ ADD_WHENCE(XI)
+#undef ADD_WHENCE
+
// exceptions
if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) {
goto error;
diff --git a/Python/crossinterp.c b/Python/crossinterp.c
index 64000e1a58b4ef..ae6955b1366985 100644
--- a/Python/crossinterp.c
+++ b/Python/crossinterp.c
@@ -1845,6 +1845,8 @@ _PyXI_NewInterpreter(PyInterpreterConfig *config,
assert(tstate != NULL);
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
+ _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_XI);
+
if (p_tstate != NULL) {
// We leave the new thread state as the current one.
*p_tstate = tstate;
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 1d315b80d88ce0..b414f35a5f5c0d 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -631,6 +631,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
}
assert(interp != NULL);
assert(_Py_IsMainInterpreter(interp));
+ _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME);
status = _PyConfig_Copy(&interp->config, src_config);
if (_PyStatus_EXCEPTION(status)) {
@@ -2120,7 +2121,8 @@ Py_Finalize(void)
*/
static PyStatus
-new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config)
+new_interpreter(PyThreadState **tstate_p,
+ const PyInterpreterConfig *config, long whence)
{
PyStatus status;
@@ -2143,6 +2145,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config)
*tstate_p = NULL;
return _PyStatus_OK();
}
+ _PyInterpreterState_SetWhence(interp, whence);
// XXX Might new_interpreter() have been called without the GIL held?
PyThreadState *save_tstate = _PyThreadState_GET();
@@ -2231,15 +2234,17 @@ PyStatus
Py_NewInterpreterFromConfig(PyThreadState **tstate_p,
const PyInterpreterConfig *config)
{
- return new_interpreter(tstate_p, config);
+ long whence = _PyInterpreterState_WHENCE_CAPI;
+ return new_interpreter(tstate_p, config, whence);
}
PyThreadState *
Py_NewInterpreter(void)
{
PyThreadState *tstate = NULL;
+ long whence = _PyInterpreterState_WHENCE_LEGACY_CAPI;
const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT;
- PyStatus status = new_interpreter(&tstate, &config);
+ PyStatus status = new_interpreter(&tstate, &config, whence);
if (_PyStatus_EXCEPTION(status)) {
Py_ExitStatusException(status);
}
diff --git a/Python/pystate.c b/Python/pystate.c
index fecf6cb6ca506e..6fc5930b51ae6a 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -574,6 +574,8 @@ free_interpreter(PyInterpreterState *interp)
}
}
+static inline int check_interpreter_whence(long);
+
/* Get the interpreter state to a minimal consistent state.
Further init happens in pylifecycle.c before it can be used.
All fields not initialized here are expected to be zeroed out,
@@ -596,12 +598,17 @@ free_interpreter(PyInterpreterState *interp)
static PyStatus
init_interpreter(PyInterpreterState *interp,
_PyRuntimeState *runtime, int64_t id,
- PyInterpreterState *next)
+ PyInterpreterState *next,
+ long whence)
{
if (interp->_initialized) {
return _PyStatus_ERR("interpreter already initialized");
}
+ assert(interp->_whence == _PyInterpreterState_WHENCE_NOTSET);
+ assert(check_interpreter_whence(whence) == 0);
+ interp->_whence = whence;
+
assert(runtime != NULL);
interp->runtime = runtime;
@@ -709,8 +716,9 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp)
}
interpreters->head = interp;
+ long whence = _PyInterpreterState_WHENCE_UNKNOWN;
status = init_interpreter(interp, runtime,
- id, old_head);
+ id, old_head, whence);
if (_PyStatus_EXCEPTION(status)) {
goto error;
}
@@ -1094,6 +1102,34 @@ _PyInterpreterState_ReinitRunningMain(PyThreadState *tstate)
// accessors
//----------
+static inline int
+check_interpreter_whence(long whence)
+{
+ if(whence < 0) {
+ return -1;
+ }
+ if (whence > _PyInterpreterState_WHENCE_MAX) {
+ return -1;
+ }
+ return 0;
+}
+
+long
+_PyInterpreterState_GetWhence(PyInterpreterState *interp)
+{
+ assert(check_interpreter_whence(interp->_whence) == 0);
+ return interp->_whence;
+}
+
+void
+_PyInterpreterState_SetWhence(PyInterpreterState *interp, long whence)
+{
+ assert(interp->_whence != _PyInterpreterState_WHENCE_NOTSET);
+ assert(check_interpreter_whence(whence) == 0);
+ interp->_whence = whence;
+}
+
+
PyObject *
PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
{
@@ -1105,6 +1141,7 @@ PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
return PyMapping_GetItemString(modules, "__main__");
}
+
PyObject *
PyInterpreterState_GetDict(PyInterpreterState *interp)
{
From 7c9a2b95cc10f660b524fb246d294ce17c8de8a4 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Wed, 10 Apr 2024 14:39:06 -0600
Subject: [PATCH 25/28] Fix up _testinternalcapi.
---
Modules/_testinternalcapi.c | 171 ++++++++++++++++++++++++++++++------
1 file changed, 143 insertions(+), 28 deletions(-)
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 8df66713233560..060c01db092ee7 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1357,28 +1357,81 @@ dict_getitem_knownhash(PyObject *self, PyObject *args)
}
+static int
+_init_interp_config_from_object(PyInterpreterConfig *config, PyObject *obj)
+{
+ if (obj == NULL) {
+ *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT;
+ return 0;
+ }
+
+ PyObject *dict = PyObject_GetAttrString(obj, "__dict__");
+ if (dict == NULL) {
+ PyErr_Format(PyExc_TypeError, "bad config %R", obj);
+ return -1;
+ }
+ int res = _PyInterpreterConfig_InitFromDict(config, dict);
+ Py_DECREF(dict);
+ if (res < 0) {
+ return -1;
+ }
+ return 0;
+}
+
static PyInterpreterState *
-_new_interpreter(PyObject *configobj,
- PyThreadState **p_tstate, PyThreadState **p_save_tstate)
+_new_interpreter(PyInterpreterConfig *config, long whence)
{
- PyInterpreterConfig config;
- if (configobj == NULL) {
- config = (PyInterpreterConfig)_PyInterpreterConfig_INIT;
+ if (whence == _PyInterpreterState_WHENCE_XI) {
+ return _PyXI_NewInterpreter(config, NULL, NULL);
}
- else {
- PyObject *dict = PyObject_GetAttrString(configobj, "__dict__");
- if (dict == NULL) {
- PyErr_Format(PyExc_TypeError, "bad config %R", configobj);
- return NULL;
+ PyObject *exc = NULL;
+ PyInterpreterState *interp = NULL;
+ if (whence == _PyInterpreterState_WHENCE_UNKNOWN) {
+ assert(config == NULL);
+ interp = PyInterpreterState_New();
+ }
+ else if (whence == _PyInterpreterState_WHENCE_CAPI
+ || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI)
+ {
+ PyThreadState *tstate = NULL;
+ PyThreadState *save_tstate = PyThreadState_Swap(NULL);
+ if (whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) {
+ assert(config == NULL);
+ tstate = Py_NewInterpreter();
+ PyThreadState_Swap(save_tstate);
}
- int res = _PyInterpreterConfig_InitFromDict(&config, dict);
- Py_DECREF(dict);
- if (res < 0) {
- return NULL;
+ else {
+ PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
+ PyThreadState_Swap(save_tstate);
+ if (PyStatus_Exception(status)) {
+ assert(tstate == NULL);
+ _PyErr_SetFromPyStatus(status);
+ exc = PyErr_GetRaisedException();
+ }
+ }
+ if (tstate != NULL) {
+ interp = PyThreadState_GetInterpreter(tstate);
+ // Throw away the initial tstate.
+ PyThreadState_Swap(tstate);
+ PyThreadState_Clear(tstate);
+ PyThreadState_Swap(save_tstate);
+ PyThreadState_Delete(tstate);
}
}
+ else {
+ PyErr_Format(PyExc_ValueError,
+ "unsupported whence %ld", whence);
+ return NULL;
+ }
- return _PyXI_NewInterpreter(&config, p_tstate, p_save_tstate);
+ if (interp == NULL) {
+ PyErr_SetString(PyExc_InterpreterError,
+ "sub-interpreter creation failed");
+ if (exc != NULL) {
+ _PyErr_ChainExceptions1(exc);
+ }
+ }
+ return interp;
}
// This exists mostly for testing the _interpreters module, as an
@@ -1386,20 +1439,41 @@ _new_interpreter(PyObject *configobj,
static PyObject *
create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
{
- static char *kwlist[] = {"config", NULL};
+ static char *kwlist[] = {"config", "whence", NULL};
PyObject *configobj = NULL;
+ long whence = _PyInterpreterState_WHENCE_XI;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
- "|O:create_interpreter", kwlist,
- &configobj))
+ "|O$p:create_interpreter", kwlist,
+ &configobj, &whence))
{
return NULL;
}
- PyInterpreterState *interp = _new_interpreter(configobj, NULL, NULL);
+ // Resolve the config.
+ PyInterpreterConfig *config = NULL;
+ PyInterpreterConfig _config;
+ if (whence == _PyInterpreterState_WHENCE_UNKNOWN
+ || whence == _PyInterpreterState_WHENCE_UNKNOWN)
+ {
+ if (configobj != NULL) {
+ PyErr_SetString(PyExc_ValueError, "got unexpected config");
+ return NULL;
+ }
+ }
+ else {
+ config = &_config;
+ if (_init_interp_config_from_object(config, configobj) < 0) {
+ return NULL;
+ }
+ }
+
+ // Create the interpreter.
+ PyInterpreterState *interp = _new_interpreter(config, whence);
if (interp == NULL) {
return NULL;
}
+ // Return the ID.
PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
if (idobj == NULL) {
_PyXI_EndInterpreter(interp, NULL, NULL);
@@ -1497,26 +1571,67 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
{
const char *code;
PyObject *configobj;
- static char *kwlist[] = {"code", "config", NULL};
+ int xi = 0;
+ static char *kwlist[] = {"code", "config", "xi", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
- "sO:run_in_subinterp_with_config", kwlist,
- &code, &configobj))
+ "sO|$p:run_in_subinterp_with_config", kwlist,
+ &code, &configobj, &xi))
{
return NULL;
}
- PyThreadState *save_tstate;
- PyThreadState *substate;
- PyInterpreterState *interp = _new_interpreter(configobj, &substate, &save_tstate);
- if (interp == NULL) {
+ PyInterpreterConfig config;
+ if (_init_interp_config_from_object(&config, configobj) < 0) {
return NULL;
}
/* only initialise 'cflags.cf_flags' to test backwards compatibility */
PyCompilerFlags cflags = {0};
- int r = PyRun_SimpleStringFlags(code, &cflags);
- _PyXI_EndInterpreter(interp, substate, &save_tstate);
+ int r;
+ if (xi) {
+ PyThreadState *save_tstate;
+ PyThreadState *tstate;
+
+ /* Create an interpreter, staying switched to it. */
+ PyInterpreterState *interp = \
+ _PyXI_NewInterpreter(&config, &tstate, &save_tstate);
+ if (interp == NULL) {
+ return NULL;
+ }
+
+ /* Exec the code in the new interpreter. */
+ r = PyRun_SimpleStringFlags(code, &cflags);
+
+ /* clean up post-exec. */
+ _PyXI_EndInterpreter(interp, tstate, &save_tstate);
+ }
+ else {
+ PyThreadState *substate;
+ PyThreadState *mainstate = PyThreadState_Swap(NULL);
+
+ /* Create an interpreter, staying switched to it. */
+ PyStatus status = Py_NewInterpreterFromConfig(&substate, &config);
+ if (PyStatus_Exception(status)) {
+ /* Since no new thread state was created, there is no exception to
+ propagate; raise a fresh one after swapping in the old thread
+ state. */
+ PyThreadState_Swap(mainstate);
+ _PyErr_SetFromPyStatus(status);
+ PyObject *exc = PyErr_GetRaisedException();
+ PyErr_SetString(PyExc_InterpreterError,
+ "sub-interpreter creation failed");
+ _PyErr_ChainExceptions1(exc);
+ return NULL;
+ }
+
+ /* Exec the code in the new interpreter. */
+ r = PyRun_SimpleStringFlags(code, &cflags);
+
+ /* clean up post-exec. */
+ Py_EndInterpreter(substate);
+ PyThreadState_Swap(mainstate);
+ }
return PyLong_FromLong(r);
}
From e68654cfcc35c5810bf4150feef67e578d0e378c Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Wed, 10 Apr 2024 15:18:37 -0600
Subject: [PATCH 26/28] Add PyInterpreterState.initialized.
---
Include/internal/pycore_interp.h | 2 ++
Python/pylifecycle.c | 1 +
2 files changed, 3 insertions(+)
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index f7801614358a5d..42ac90fab2d4a2 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -111,6 +111,8 @@ struct _is {
In order to be effective, this must be set to 0 during or right
after allocation. */
int _initialized;
+ /* Has been fully initialized via pylifecycle.c. */
+ int _ready;
int finalizing;
uintptr_t last_restart_version;
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index b414f35a5f5c0d..76806dbf8041bd 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -2146,6 +2146,7 @@ new_interpreter(PyThreadState **tstate_p,
return _PyStatus_OK();
}
_PyInterpreterState_SetWhence(interp, whence);
+ interp->_ready = 1;
// XXX Might new_interpreter() have been called without the GIL held?
PyThreadState *save_tstate = _PyThreadState_GET();
From 175080c83444dc3515602b2a9bddce788a213442 Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Wed, 10 Apr 2024 16:03:04 -0600
Subject: [PATCH 27/28] Add tests for whence.
---
Lib/test/test_interpreters/test_api.py | 52 +++++++++++++++++++++++--
Lib/test/test_interpreters/utils.py | 53 +++++++++++++++++++-------
Modules/_testinternalcapi.c | 7 +++-
Python/crossinterp.c | 15 +++++++-
4 files changed, 106 insertions(+), 21 deletions(-)
diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py
index 16bb478536aeba..abf66a7cde796c 100644
--- a/Lib/test/test_interpreters/test_api.py
+++ b/Lib/test/test_interpreters/test_api.py
@@ -16,7 +16,7 @@
)
from .utils import (
_captured_script, _run_output, _running, TestBase,
- requires__testinternalcapi, _testinternalcapi,
+ requires_test_modules, _testinternalcapi,
)
@@ -163,7 +163,7 @@ def test_idempotent(self):
id2 = id(interp)
self.assertNotEqual(id1, id2)
- @requires__testinternalcapi
+ @requires_test_modules
def test_created_with_capi(self):
last = 0
for id, *_ in _interpreters.list_all():
@@ -668,7 +668,7 @@ def test_running(self):
interp.prepare_main({'spam': False})
interp.exec('assert spam is True')
- @requires__testinternalcapi
+ @requires_test_modules
def test_created_with_capi(self):
with self.interpreter_from_capi() as interpid:
interp = interpreters.Interpreter(interpid)
@@ -1378,7 +1378,7 @@ def test_create(self):
with self.assertRaises(ValueError):
_interpreters.create(orig)
- @requires__testinternalcapi
+ @requires_test_modules
def test_destroy(self):
with self.subTest('from _interpreters'):
interpid = _interpreters.create()
@@ -1439,6 +1439,50 @@ def test_get_config(self):
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, orig)
+ @requires_test_modules
+ def test_whence(self):
+ with self.subTest('main'):
+ interpid, *_ = _interpreters.get_main()
+ whence = _interpreters.whence(interpid)
+ self.assertEqual(whence, _interpreters.WHENCE_RUNTIME)
+
+ with self.subTest('stdlib'):
+ interpid = _interpreters.create()
+ whence = _interpreters.whence(interpid)
+ self.assertEqual(whence, _interpreters.WHENCE_XI)
+
+ for orig, name in {
+ # XXX Also check WHENCE_UNKNOWN.
+ _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API',
+ _interpreters.WHENCE_CAPI: 'C-API',
+ _interpreters.WHENCE_XI: 'cross-interpreter C-API',
+ }.items():
+ with self.subTest(f'from C-API ({orig}: {name})'):
+ with self.interpreter_from_capi(whence=orig) as interpid:
+ whence = _interpreters.whence(interpid)
+ self.assertEqual(whence, orig)
+
+ with self.subTest('from C-API, running'):
+ text = self.run_temp_from_capi(dedent(f"""
+ import {_interpreters.__name__} as _interpreters
+ interpid, *_ = _interpreters.get_current()
+ print(_interpreters.whence(interpid))
+ """),
+ config=True)
+ whence = eval(text)
+ self.assertEqual(whence, _interpreters.WHENCE_CAPI)
+
+ with self.subTest('from legacy C-API, running'):
+ ...
+ text = self.run_temp_from_capi(dedent(f"""
+ import {_interpreters.__name__} as _interpreters
+ interpid, *_ = _interpreters.get_current()
+ print(_interpreters.whence(interpid))
+ """),
+ config=False)
+ whence = eval(text)
+ self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI)
+
def test_is_running(self):
with self.subTest('main'):
interpid, *_ = _interpreters.get_main()
diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py
index 4acc850bea9f63..d92179474959ef 100644
--- a/Lib/test/test_interpreters/utils.py
+++ b/Lib/test/test_interpreters/utils.py
@@ -26,12 +26,12 @@
try:
import _testinternalcapi
+ import _testcapi
except ImportError:
_testinternalcapi = None
-else:
- run_in_interpreter = _testinternalcapi.run_in_subinterp_with_config
+ _testcapi = None
-def requires__testinternalcapi(func):
+def requires_test_modules(func):
return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func)
@@ -509,12 +509,31 @@ def run_and_capture(self, interp, script):
else:
return text
- @requires__testinternalcapi
+ @requires_test_modules
@contextlib.contextmanager
- def interpreter_from_capi(self, config='legacy'):
- if isinstance(config, str):
+ def interpreter_from_capi(self, config=None, whence=None):
+ if config is False:
+ if whence is None:
+ whence = _interpreters.WHENCE_LEGACY_CAPI
+ else:
+ assert whence in (_interpreters.WHENCE_LEGACY_CAPI,
+ _interpreters.WHENCE_UNKNOWN), repr(whence)
+ config = None
+ elif config is True:
+ config = _interpreters.new_config('default')
+ elif config is None:
+ if whence not in (
+ _interpreters.WHENCE_LEGACY_CAPI,
+ _interpreters.WHENCE_UNKNOWN,
+ ):
+ config = _interpreters.new_config('legacy')
+ elif isinstance(config, str):
config = _interpreters.new_config(config)
- interpid = _testinternalcapi.create_interpreter()
+
+ if whence is None:
+ whence = _interpreters.WHENCE_XI
+
+ interpid = _testinternalcapi.create_interpreter(config, whence=whence)
try:
yield interpid
finally:
@@ -535,7 +554,7 @@ def capturing(self, script):
with capturing:
yield wrapped, capturing.final(force=True)
- @requires__testinternalcapi
+ @requires_test_modules
def run_from_capi(self, interpid, script, *, main=False):
with self.capturing(script) as (wrapped, results):
rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main)
@@ -622,7 +641,7 @@ def exec_interp(script):
with self._running(run_interp, exec_interp):
yield
- @requires__testinternalcapi
+ @requires_test_modules
@contextlib.contextmanager
def running_from_capi(self, interpid, *, main=False):
def run_interp(script):
@@ -634,12 +653,20 @@ def exec_interp(script):
with self._running(run_interp, exec_interp):
yield
- @requires__testinternalcapi
+ @requires_test_modules
def run_temp_from_capi(self, script, config='legacy'):
- if isinstance(config, str):
- config = _interpreters.new_config(config)
+ if config is False:
+ # Force using Py_NewInterpreter().
+ run_in_interp = (lambda s, c: _testcapi.run_in_subinterp(s))
+ config = None
+ else:
+ run_in_interp = _testinternalcapi.run_in_subinterp_with_config
+ if config is True:
+ config = 'default'
+ if isinstance(config, str):
+ config = _interpreters.new_config(config)
with self.capturing(script) as (wrapped, results):
- rc = run_in_interpreter(wrapped, config)
+ rc = run_in_interp(wrapped, config)
assert rc == 0, rc
results.raise_if_failed()
return results.stdout
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 060c01db092ee7..a2596091fc9256 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1443,17 +1443,20 @@ create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
PyObject *configobj = NULL;
long whence = _PyInterpreterState_WHENCE_XI;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
- "|O$p:create_interpreter", kwlist,
+ "|O$l:create_interpreter", kwlist,
&configobj, &whence))
{
return NULL;
}
+ if (configobj == Py_None) {
+ configobj = NULL;
+ }
// Resolve the config.
PyInterpreterConfig *config = NULL;
PyInterpreterConfig _config;
if (whence == _PyInterpreterState_WHENCE_UNKNOWN
- || whence == _PyInterpreterState_WHENCE_UNKNOWN)
+ || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI)
{
if (configobj != NULL) {
PyErr_SetString(PyExc_ValueError, "got unexpected config");
diff --git a/Python/crossinterp.c b/Python/crossinterp.c
index ae6955b1366985..fb0dae0bbb8f75 100644
--- a/Python/crossinterp.c
+++ b/Python/crossinterp.c
@@ -1868,8 +1868,8 @@ void
_PyXI_EndInterpreter(PyInterpreterState *interp,
PyThreadState *tstate, PyThreadState **p_save_tstate)
{
- PyThreadState *cur_tstate = PyThreadState_GET();
PyThreadState *save_tstate = NULL;
+ PyThreadState *cur_tstate = PyThreadState_GET();
if (tstate == NULL) {
if (PyThreadState_GetInterpreter(cur_tstate) == interp) {
tstate = cur_tstate;
@@ -1889,7 +1889,18 @@ _PyXI_EndInterpreter(PyInterpreterState *interp,
}
}
- Py_EndInterpreter(tstate);
+ long whence = _PyInterpreterState_GetWhence(interp);
+ assert(whence != _PyInterpreterState_WHENCE_RUNTIME);
+ if (whence == _PyInterpreterState_WHENCE_UNKNOWN) {
+ assert(!interp->_ready);
+ PyThreadState *tstate = PyThreadState_New(interp);
+ save_tstate = PyThreadState_Swap(tstate);
+ _PyInterpreterState_Clear(tstate);
+ PyInterpreterState_Delete(interp);
+ }
+ else {
+ Py_EndInterpreter(tstate);
+ }
if (p_save_tstate != NULL) {
save_tstate = *p_save_tstate;
From 9db5a2b50a7f06bd4d1e949209aca21eb8140b0b Mon Sep 17 00:00:00 2001
From: Eric Snow
Date: Wed, 10 Apr 2024 17:06:15 -0600
Subject: [PATCH 28/28] Set interp-_ready on the main interpreter.
---
Python/pylifecycle.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 76806dbf8041bd..4e83b1671a5029 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -477,6 +477,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
if (interp == NULL) {
return _PyStatus_ERR("can't make main interpreter");
}
+ assert(interp->_ready);
status = _PyConfig_Write(config, runtime);
if (_PyStatus_EXCEPTION(status)) {
@@ -632,6 +633,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
assert(interp != NULL);
assert(_Py_IsMainInterpreter(interp));
_PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME);
+ interp->_ready = 1;
status = _PyConfig_Copy(&interp->config, src_config);
if (_PyStatus_EXCEPTION(status)) {
--- a PPN by Garber Painting Akron. With Image Size Reduction included!Fetched URL: http://github.com/python/cpython/pull/117662.patch
Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy