From bdaef6b92c0be66cb60f22566fcba1b54237d264 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Mar 2024 16:47:42 -0700 Subject: [PATCH 01/19] Fix a comment. --- Lib/test/test_interpreters/test_queues.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index d16d294b82d044..8ab9ebb354712a 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -28,9 +28,9 @@ def tearDown(self): class LowLevelTests(TestBase): - # The behaviors in the low-level module is important in as much - # as it is exercised by the high-level module. Therefore the - # most # important testing happens in the high-level tests. + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. # These low-level tests cover corner cases that are not # encountered by the high-level module, thus they # mostly shouldn't matter as much. From 6342635375bfa24bb30e1bfae80fead50a5916b9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 14:50:00 -0600 Subject: [PATCH 02/19] Add PyInterpreterConfig helpers. --- Include/internal/pycore_pylifecycle.h | 13 ++ Python/initconfig.c | 244 +++++++++++++++++++++++++- Python/pystate.c | 22 +++ 3 files changed, 275 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index c675098685764c..5f24324830f06f 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -116,6 +116,19 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); // Export for special main.c string compiling with source tracebacks int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + +/* interpreter config */ + +extern int _PyInterpreterState_ResolveConfig( + PyInterpreterState *, + PyInterpreterConfig *); +extern PyObject * _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +extern int _PyInterpreterConfig_FromDict(PyObject *, PyInterpreterConfig *); +extern int _PyInterpreterConfig_UpdateFromDict( + PyObject *, + PyInterpreterConfig *); + + #ifdef __cplusplus } #endif diff --git a/Python/initconfig.c b/Python/initconfig.c index 215d6a1d4e0dba..9f2e42dfd87b04 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1098,15 +1098,30 @@ _PyConfig_AsDict(const PyConfig *config) } -static PyObject* -config_dict_get(PyObject *dict, const char *name) +static inline int +_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) { PyObject *item; if (PyDict_GetItemStringRef(dict, name, &item) < 0) { - return NULL; + return -1; } if (item == NULL) { - PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + // We do not set an exception. + return -1; + } + *p_item = item; + return 0; +} + + +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + } return NULL; } return item; @@ -3249,3 +3264,224 @@ _Py_DumpPathConfig(PyThreadState *tstate) _PyErr_SetRaisedException(tstate, exc); } + + +/* PyInterpreterConfig + + This isn't exactly the right place for API related to PyInterpreterConfig, + but neither pylifecycle.c nor pystate.c are quite right either. + For now, this is as good a place as any. +*/ + +static const char * +gil_flag_to_str(int flag) +{ + switch (flag) { + case PyInterpreterConfig_DEFAULT_GIL: + return "default"; + case PyInterpreterConfig_SHARED_GIL: + return "shared"; + case PyInterpreterConfig_OWN_GIL: + return "own"; + default: + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return NULL; + } +} + +static int +gil_flag_from_str(const char *str, int *p_flag) +{ + int flag; + if (str == NULL || strcmp(str, "") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "default") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "shared") == 0) { + flag = PyInterpreterConfig_SHARED_GIL; + } + else if (strcmp(str, "own") == 0) { + flag = PyInterpreterConfig_OWN_GIL; + } + else { + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return -1; + } + *p_flag = flag; + return 0; +} + +PyObject * +_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define ADD(NAME, OBJ) \ + do { \ + int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ + Py_DECREF(OBJ); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) +#define ADD_BOOL(FIELD) \ + ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) +#define ADD_STR(FIELD, STR) \ + do { \ + if (STR == NULL) { \ + goto error; \ + } \ + PyObject *obj = PyUnicode_FromString(#FIELD); \ + if (obj == NULL) { \ + goto error; \ + } \ + ADD(#FIELD, obj); \ + } while (0) + + ADD_BOOL(use_main_obmalloc); + ADD_BOOL(allow_fork); + ADD_BOOL(allow_exec); + ADD_BOOL(allow_threads); + ADD_BOOL(allow_daemon_threads); + ADD_BOOL(check_multi_interp_extensions); + + ADD_STR(gil, gil_flag_to_str(config->gil)); + +#undef ADD_STR +#undef ADD_BOOL +#undef ADD + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} + +static int +_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + int flag = PyObject_IsTrue(item); + Py_DECREF(item); + if (flag < 0) { + return -1; + } + *p_flag = flag; + return 0; +} + +static int +_config_dict_copy_str(PyObject *dict, const char *name, + char *buf, size_t bufsize) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + if (!PyUnicode_Check(item)) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); + buf[bufsize-1] = '\0'; + Py_DECREF(item); + return 0; +} + +static int +interp_config_from_dict(PyObject *dict, PyInterpreterConfig *config, + bool missing_allowed) +{ + Py_ssize_t unused = PyDict_GET_SIZE(dict); + +#define CHECK(NAME) \ + do { \ + if (!PyErr_Occurred()) { \ + if (!missing_allowed) { \ + (void)config_dict_get(dict, NAME); \ + assert(PyErr_Occurred()); \ + return -1; \ + } \ + } \ + } while (0) +#define COPY_BOOL(FIELD) \ + do { \ + int flag; \ + if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ + CHECK(#FIELD); \ + } \ + else { \ + config->FIELD = flag; \ + unused -= 1; \ + } \ + } while (0) + + COPY_BOOL(use_main_obmalloc); + COPY_BOOL(allow_fork); + COPY_BOOL(allow_exec); + COPY_BOOL(allow_threads); + COPY_BOOL(allow_daemon_threads); + COPY_BOOL(check_multi_interp_extensions); + + // PyInterpreterConfig.gil + char buf[20]; + if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { + CHECK("gil"); + } + else { + int flag; + if (gil_flag_from_str(buf, &flag) < 0) { + return -1; + } + config->gil = flag; + unused -= 1; + } + +#undef COPY_BOOL +#undef CHECK + + if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "dict as %d extra items", unused); + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, false) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_UpdateFromDict(PyObject *dict, PyInterpreterConfig *config) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, true) < 0) { + return -1; + } + return 0; +} diff --git a/Python/pystate.c b/Python/pystate.c index 47d327ae28933b..4a2106f19701c8 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1122,6 +1122,28 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) return interp->dict; } +int +_PyInterpreterState_ResolveConfig(PyInterpreterState *interp, + PyInterpreterConfig *config) +{ + // Populate the config by re-constructing the values from the interpreter. + *config = (PyInterpreterConfig){ +#define FLAG(flag) \ + (interp->feature_flags & Py_RTFLAGS_ ## flag) + .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), + .allow_fork = FLAG(FORK), + .allow_exec = FLAG(EXEC), + .allow_threads = FLAG(THREADS), + .allow_daemon_threads = FLAG(DAEMON_THREADS), + .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), +#undef FLAG + .gil = interp->ceval.own_gil + ? PyInterpreterConfig_OWN_GIL + : PyInterpreterConfig_SHARED_GIL, + }; + return 0; +} + //---------- // interp ID From c48dd00f4ceb53e3c86da3782b90b5e8c160ce60 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 17:48:22 -0600 Subject: [PATCH 03/19] Add config helpers to _testinternalcapi. --- Include/internal/pycore_pylifecycle.h | 11 ++-- Modules/_testinternalcapi.c | 91 +++++++++++++++++++++++++++ Python/initconfig.c | 2 +- 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 5f24324830f06f..a230cd2c5b927a 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -119,14 +119,17 @@ int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCo /* interpreter config */ -extern int _PyInterpreterState_ResolveConfig( +// Export for _testinternalcapi shared extension +PyAPI_FUNC(int) _PyInterpreterState_ResolveConfig( PyInterpreterState *, PyInterpreterConfig *); -extern PyObject * _PyInterpreterConfig_AsDict(PyInterpreterConfig *); -extern int _PyInterpreterConfig_FromDict(PyObject *, PyInterpreterConfig *); -extern int _PyInterpreterConfig_UpdateFromDict( +PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_FromDict( PyObject *, PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( + PyInterpreterConfig *, + PyObject *); #ifdef __cplusplus diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e1717f7a66b1de..1e3cee21642408 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,10 +23,12 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pylifecycle.h" // _PyInterpreterState_ResolveConfig() #include "pycore_pystate.h" // _PyThreadState_GET() #include "clinic/_testinternalcapi.c.h" @@ -829,6 +831,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } +// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -1376,6 +1379,91 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } +static PyObject * +get_interpreter_config(PyObject *self, PyObject *args) +{ + PyObject *idobj = NULL; + if (!PyArg_ParseTuple(args, "|O:get_interpreter_config", &idobj)) { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterState_ResolveConfig(interp, &config) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static PyObject * +new_interpreter_config(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *initialized = NULL; + PyObject *overrides = NULL; + static char *kwlist[] = {"initialized", "overrides", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|sO:new_interpreter_config", kwlist, + &initialized, &overrides)) + { + return NULL; + } + if (initialized == NULL + || strcmp(initialized, "") == 0 + || strcmp(initialized, "default") == 0) + { + initialized = "isolated"; + } + + PyInterpreterConfig config; + if (strcmp(initialized, "isolated") == 0) { + config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(initialized, "legacy") == 0) { + config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(initialized, "empty") == 0) { + config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported initialized arg '%s'", initialized); + return NULL; + } + + if (overrides != NULL) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + + /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1858,6 +1946,9 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"get_interpreter_config", get_interpreter_config, METH_VARARGS}, + {"new_interpreter_config", _PyCFunction_CAST(new_interpreter_config), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, diff --git a/Python/initconfig.c b/Python/initconfig.c index 9f2e42dfd87b04..3ef47e9c101423 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -3474,7 +3474,7 @@ _PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) } int -_PyInterpreterConfig_UpdateFromDict(PyObject *dict, PyInterpreterConfig *config) +_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) { if (!PyDict_Check(dict)) { PyErr_SetString(PyExc_TypeError, "dict expected"); From 0796fe99ad1a4f140f391dba57fa7721a5362553 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 19:00:56 -0600 Subject: [PATCH 04/19] Use the new helpers in run_in_subinterp_with_config(). --- Lib/test/support/__init__.py | 15 +++++- Lib/test/test_import/__init__.py | 12 ++++- Modules/_testinternalcapi.c | 80 +++++++------------------------- Python/initconfig.c | 39 ++++++++++++---- 4 files changed, 70 insertions(+), 76 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a1c7987fa0db47..e99555efca2622 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1728,8 +1728,19 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): import _testinternalcapi if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testinternalcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + else: + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 4deed7f3ba2522..81ec700d9755ce 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1823,15 +1823,19 @@ def check_compatible_fresh(self, name, *, strict=False, isolated=False): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=strict, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert ( {name!r} in sys.builtin_module_names or {name!r} not in sys.modules ), repr({name!r}) + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) @@ -1847,12 +1851,16 @@ def check_incompatible_fresh(self, name, *, isolated=False): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=True, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert {name!r} not in sys.modules, {name!r} + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1e3cee21642408..aff6af9e47c75d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1469,78 +1469,32 @@ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) { const char *code; - int use_main_obmalloc = -1; - int allow_fork = -1; - int allow_exec = -1; - int allow_threads = -1; - int allow_daemon_threads = -1; - int check_multi_interp_extensions = -1; - int gil = -1; - int r; - PyThreadState *substate, *mainstate; - /* only initialise 'cflags.cf_flags' to test backwards compatibility */ - PyCompilerFlags cflags = {0}; - - static char *kwlist[] = {"code", - "use_main_obmalloc", - "allow_fork", - "allow_exec", - "allow_threads", - "allow_daemon_threads", - "check_multi_interp_extensions", - "gil", - NULL}; + PyObject *configobj; + static char *kwlist[] = {"code", "config", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppi:run_in_subinterp_with_config", kwlist, - &code, &use_main_obmalloc, - &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions, - &gil)) { - return NULL; - } - if (use_main_obmalloc < 0) { - PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); - return NULL; - } - if (allow_fork < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_fork"); - return NULL; - } - if (allow_exec < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_exec"); - return NULL; - } - if (allow_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_threads"); - return NULL; - } - if (gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing gil"); + "sO:run_in_subinterp_with_config", kwlist, + &code, &configobj)) + { return NULL; } - if (allow_daemon_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); + + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); return NULL; } - if (check_multi_interp_extensions < 0) { - PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + PyInterpreterConfig config = {0}; + int res = _PyInterpreterConfig_FromDict(dict, &config); + Py_DECREF(dict); + if (res < 0) { return NULL; } - mainstate = PyThreadState_Get(); + PyThreadState *mainstate = PyThreadState_Get(); PyThreadState_Swap(NULL); - const PyInterpreterConfig config = { - .use_main_obmalloc = use_main_obmalloc, - .allow_fork = allow_fork, - .allow_exec = allow_exec, - .allow_threads = allow_threads, - .allow_daemon_threads = allow_daemon_threads, - .check_multi_interp_extensions = check_multi_interp_extensions, - .gil = gil, - }; + PyThreadState *substate; PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { /* Since no new thread state was created, there is no exception to @@ -1554,7 +1508,9 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } assert(substate != NULL); - r = PyRun_SimpleStringFlags(code, &cflags); + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + int r = PyRun_SimpleStringFlags(code, &cflags); Py_EndInterpreter(substate); PyThreadState_Swap(mainstate); diff --git a/Python/initconfig.c b/Python/initconfig.c index 3ef47e9c101423..da5e523f0dfe83 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -3401,18 +3401,27 @@ _config_dict_copy_str(PyObject *dict, const char *name, } static int -interp_config_from_dict(PyObject *dict, PyInterpreterConfig *config, +interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, bool missing_allowed) { - Py_ssize_t unused = PyDict_GET_SIZE(dict); + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return -1; + } + if (PyDict_Update(dict, origdict) < 0) { + goto error; + } #define CHECK(NAME) \ do { \ - if (!PyErr_Occurred()) { \ + if (PyErr_Occurred()) { \ + goto error; \ + } \ + else { \ if (!missing_allowed) { \ (void)config_dict_get(dict, NAME); \ assert(PyErr_Occurred()); \ - return -1; \ + goto error; \ } \ } \ } while (0) @@ -3424,7 +3433,7 @@ interp_config_from_dict(PyObject *dict, PyInterpreterConfig *config, } \ else { \ config->FIELD = flag; \ - unused -= 1; \ + (void)PyDict_PopString(dict, #FIELD, NULL); \ } \ } while (0) @@ -3443,21 +3452,31 @@ interp_config_from_dict(PyObject *dict, PyInterpreterConfig *config, else { int flag; if (gil_flag_from_str(buf, &flag) < 0) { - return -1; + goto error; } config->gil = flag; - unused -= 1; + (void)PyDict_PopString(dict, "gil", NULL); } #undef COPY_BOOL #undef CHECK - if (unused > 0) { + Py_ssize_t unused = PyDict_GET_SIZE(dict); + if (unused == 1) { PyErr_Format(PyExc_ValueError, - "dict as %d extra items", unused); - return -1; + "config dict has 1 extra item (%R)", dict); + goto error; + } + else if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "config dict has %d extra items (%R)", unused, dict); + goto error; } return 0; + +error: + Py_DECREF(dict); + return -1; } int From 8993b41757dd2c3a250b3bb655d0ee28e55af22d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 11:17:13 -0600 Subject: [PATCH 05/19] Move the PyInterpreterConfig utils to their own file. --- Makefile.pre.in | 5 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/config_common.h | 36 ++++ Python/initconfig.c | 280 +------------------------ Python/interpconfig.c | 241 +++++++++++++++++++++ 8 files changed, 293 insertions(+), 277 deletions(-) create mode 100644 Python/config_common.h create mode 100644 Python/interpconfig.c diff --git a/Makefile.pre.in b/Makefile.pre.in index c454f31aae1e57..1c34adce9a27f4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -434,6 +434,7 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/interpconfig.o \ Python/instrumentation.o \ Python/intrinsics.o \ Python/jit.o \ @@ -1680,6 +1681,10 @@ Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $ Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h +Python/initconfig.o: $(srcdir)/Python/initconfig.c $(srcdir)/Python/config_common.h + +Python/interpconfig.o: $(srcdir)/Python/interpconfig.c $(srcdir)/Python/config_common.h + Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ -DSOABI='"$(SOABI)"' \ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 82471e0f140ec3..9c82fcf021bb55 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -222,6 +222,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 97c52fdadf7c05..63b033a0350b20 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -229,6 +229,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c944bbafdba7e5..97f779515a86d2 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -586,6 +586,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0afad125ce1e97..8e1bd456cf5352 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1340,6 +1340,9 @@ Python + + Python + Source Files diff --git a/Python/config_common.h b/Python/config_common.h new file mode 100644 index 00000000000000..e749bd4bf0dc68 --- /dev/null +++ b/Python/config_common.h @@ -0,0 +1,36 @@ + +static inline int +_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) +{ + PyObject *item; + if (PyDict_GetItemStringRef(dict, name, &item) < 0) { + return -1; + } + if (item == NULL) { + // We do not set an exception. + return -1; + } + *p_item = item; + return 0; +} + + +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + } + return NULL; + } + return item; +} + + +static void +config_dict_invalid_type(const char *name) +{ + PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); +} diff --git a/Python/initconfig.c b/Python/initconfig.c index da5e523f0dfe83..d91a8199b544dc 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -24,6 +24,9 @@ # endif #endif +#include "config_common.h" + + /* --- PyConfig spec ---------------------------------------------- */ typedef enum { @@ -1098,36 +1101,6 @@ _PyConfig_AsDict(const PyConfig *config) } -static inline int -_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) -{ - PyObject *item; - if (PyDict_GetItemStringRef(dict, name, &item) < 0) { - return -1; - } - if (item == NULL) { - // We do not set an exception. - return -1; - } - *p_item = item; - return 0; -} - - -static PyObject* -config_dict_get(PyObject *dict, const char *name) -{ - PyObject *item; - if (_config_dict_get(dict, name, &item) < 0) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, "missing config key: %s", name); - } - return NULL; - } - return item; -} - - static void config_dict_invalid_value(const char *name) { @@ -1135,13 +1108,6 @@ config_dict_invalid_value(const char *name) } -static void -config_dict_invalid_type(const char *name) -{ - PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); -} - - static int config_dict_get_int(PyObject *dict, const char *name, int *result) { @@ -3264,243 +3230,3 @@ _Py_DumpPathConfig(PyThreadState *tstate) _PyErr_SetRaisedException(tstate, exc); } - - -/* PyInterpreterConfig - - This isn't exactly the right place for API related to PyInterpreterConfig, - but neither pylifecycle.c nor pystate.c are quite right either. - For now, this is as good a place as any. -*/ - -static const char * -gil_flag_to_str(int flag) -{ - switch (flag) { - case PyInterpreterConfig_DEFAULT_GIL: - return "default"; - case PyInterpreterConfig_SHARED_GIL: - return "shared"; - case PyInterpreterConfig_OWN_GIL: - return "own"; - default: - PyErr_SetString(PyExc_SystemError, - "invalid interpreter config 'gil' value"); - return NULL; - } -} - -static int -gil_flag_from_str(const char *str, int *p_flag) -{ - int flag; - if (str == NULL || strcmp(str, "") == 0) { - flag = PyInterpreterConfig_DEFAULT_GIL; - } - else if (strcmp(str, "default") == 0) { - flag = PyInterpreterConfig_DEFAULT_GIL; - } - else if (strcmp(str, "shared") == 0) { - flag = PyInterpreterConfig_SHARED_GIL; - } - else if (strcmp(str, "own") == 0) { - flag = PyInterpreterConfig_OWN_GIL; - } - else { - PyErr_SetString(PyExc_SystemError, - "invalid interpreter config 'gil' value"); - return -1; - } - *p_flag = flag; - return 0; -} - -PyObject * -_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) -{ - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - -#define ADD(NAME, OBJ) \ - do { \ - int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ - Py_DECREF(OBJ); \ - if (res < 0) { \ - goto error; \ - } \ - } while (0) -#define ADD_BOOL(FIELD) \ - ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) -#define ADD_STR(FIELD, STR) \ - do { \ - if (STR == NULL) { \ - goto error; \ - } \ - PyObject *obj = PyUnicode_FromString(#FIELD); \ - if (obj == NULL) { \ - goto error; \ - } \ - ADD(#FIELD, obj); \ - } while (0) - - ADD_BOOL(use_main_obmalloc); - ADD_BOOL(allow_fork); - ADD_BOOL(allow_exec); - ADD_BOOL(allow_threads); - ADD_BOOL(allow_daemon_threads); - ADD_BOOL(check_multi_interp_extensions); - - ADD_STR(gil, gil_flag_to_str(config->gil)); - -#undef ADD_STR -#undef ADD_BOOL -#undef ADD - - return dict; - -error: - Py_DECREF(dict); - return NULL; -} - -static int -_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) -{ - PyObject *item; - if (_config_dict_get(dict, name, &item) < 0) { - return -1; - } - int flag = PyObject_IsTrue(item); - Py_DECREF(item); - if (flag < 0) { - return -1; - } - *p_flag = flag; - return 0; -} - -static int -_config_dict_copy_str(PyObject *dict, const char *name, - char *buf, size_t bufsize) -{ - PyObject *item; - if (_config_dict_get(dict, name, &item) < 0) { - return -1; - } - if (!PyUnicode_Check(item)) { - Py_DECREF(item); - config_dict_invalid_type(name); - return -1; - } - strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); - buf[bufsize-1] = '\0'; - Py_DECREF(item); - return 0; -} - -static int -interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, - bool missing_allowed) -{ - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return -1; - } - if (PyDict_Update(dict, origdict) < 0) { - goto error; - } - -#define CHECK(NAME) \ - do { \ - if (PyErr_Occurred()) { \ - goto error; \ - } \ - else { \ - if (!missing_allowed) { \ - (void)config_dict_get(dict, NAME); \ - assert(PyErr_Occurred()); \ - goto error; \ - } \ - } \ - } while (0) -#define COPY_BOOL(FIELD) \ - do { \ - int flag; \ - if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ - CHECK(#FIELD); \ - } \ - else { \ - config->FIELD = flag; \ - (void)PyDict_PopString(dict, #FIELD, NULL); \ - } \ - } while (0) - - COPY_BOOL(use_main_obmalloc); - COPY_BOOL(allow_fork); - COPY_BOOL(allow_exec); - COPY_BOOL(allow_threads); - COPY_BOOL(allow_daemon_threads); - COPY_BOOL(check_multi_interp_extensions); - - // PyInterpreterConfig.gil - char buf[20]; - if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { - CHECK("gil"); - } - else { - int flag; - if (gil_flag_from_str(buf, &flag) < 0) { - goto error; - } - config->gil = flag; - (void)PyDict_PopString(dict, "gil", NULL); - } - -#undef COPY_BOOL -#undef CHECK - - Py_ssize_t unused = PyDict_GET_SIZE(dict); - if (unused == 1) { - PyErr_Format(PyExc_ValueError, - "config dict has 1 extra item (%R)", dict); - goto error; - } - else if (unused > 0) { - PyErr_Format(PyExc_ValueError, - "config dict has %d extra items (%R)", unused, dict); - goto error; - } - return 0; - -error: - Py_DECREF(dict); - return -1; -} - -int -_PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) -{ - if (!PyDict_Check(dict)) { - PyErr_SetString(PyExc_TypeError, "dict expected"); - return -1; - } - if (interp_config_from_dict(dict, config, false) < 0) { - return -1; - } - return 0; -} - -int -_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) -{ - if (!PyDict_Check(dict)) { - PyErr_SetString(PyExc_TypeError, "dict expected"); - return -1; - } - if (interp_config_from_dict(dict, config, true) < 0) { - return -1; - } - return 0; -} diff --git a/Python/interpconfig.c b/Python/interpconfig.c new file mode 100644 index 00000000000000..601fc6744567e8 --- /dev/null +++ b/Python/interpconfig.c @@ -0,0 +1,241 @@ +/* PyInterpreterConfig API */ + +#include "Python.h" +#include "pycore_pylifecycle.h" + +#include + +#include "config_common.h" + + +static const char * +gil_flag_to_str(int flag) +{ + switch (flag) { + case PyInterpreterConfig_DEFAULT_GIL: + return "default"; + case PyInterpreterConfig_SHARED_GIL: + return "shared"; + case PyInterpreterConfig_OWN_GIL: + return "own"; + default: + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return NULL; + } +} + +static int +gil_flag_from_str(const char *str, int *p_flag) +{ + int flag; + if (str == NULL || strcmp(str, "") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "default") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "shared") == 0) { + flag = PyInterpreterConfig_SHARED_GIL; + } + else if (strcmp(str, "own") == 0) { + flag = PyInterpreterConfig_OWN_GIL; + } + else { + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return -1; + } + *p_flag = flag; + return 0; +} + +PyObject * +_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define ADD(NAME, OBJ) \ + do { \ + int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ + Py_DECREF(OBJ); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) +#define ADD_BOOL(FIELD) \ + ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) +#define ADD_STR(FIELD, STR) \ + do { \ + if (STR == NULL) { \ + goto error; \ + } \ + PyObject *obj = PyUnicode_FromString(#FIELD); \ + if (obj == NULL) { \ + goto error; \ + } \ + ADD(#FIELD, obj); \ + } while (0) + + ADD_BOOL(use_main_obmalloc); + ADD_BOOL(allow_fork); + ADD_BOOL(allow_exec); + ADD_BOOL(allow_threads); + ADD_BOOL(allow_daemon_threads); + ADD_BOOL(check_multi_interp_extensions); + + ADD_STR(gil, gil_flag_to_str(config->gil)); + +#undef ADD_STR +#undef ADD_BOOL +#undef ADD + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} + +static int +_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + int flag = PyObject_IsTrue(item); + Py_DECREF(item); + if (flag < 0) { + return -1; + } + *p_flag = flag; + return 0; +} + +static int +_config_dict_copy_str(PyObject *dict, const char *name, + char *buf, size_t bufsize) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + if (!PyUnicode_Check(item)) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); + buf[bufsize-1] = '\0'; + Py_DECREF(item); + return 0; +} + +static int +interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, + bool missing_allowed) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return -1; + } + if (PyDict_Update(dict, origdict) < 0) { + goto error; + } + +#define CHECK(NAME) \ + do { \ + if (PyErr_Occurred()) { \ + goto error; \ + } \ + else { \ + if (!missing_allowed) { \ + (void)config_dict_get(dict, NAME); \ + assert(PyErr_Occurred()); \ + goto error; \ + } \ + } \ + } while (0) +#define COPY_BOOL(FIELD) \ + do { \ + int flag; \ + if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ + CHECK(#FIELD); \ + } \ + else { \ + config->FIELD = flag; \ + (void)PyDict_PopString(dict, #FIELD, NULL); \ + } \ + } while (0) + + COPY_BOOL(use_main_obmalloc); + COPY_BOOL(allow_fork); + COPY_BOOL(allow_exec); + COPY_BOOL(allow_threads); + COPY_BOOL(allow_daemon_threads); + COPY_BOOL(check_multi_interp_extensions); + + // PyInterpreterConfig.gil + char buf[20]; + if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { + CHECK("gil"); + } + else { + int flag; + if (gil_flag_from_str(buf, &flag) < 0) { + goto error; + } + config->gil = flag; + (void)PyDict_PopString(dict, "gil", NULL); + } + +#undef COPY_BOOL +#undef CHECK + + Py_ssize_t unused = PyDict_GET_SIZE(dict); + if (unused == 1) { + PyErr_Format(PyExc_ValueError, + "config dict has 1 extra item (%R)", dict); + goto error; + } + else if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "config dict has %d extra items (%R)", unused, dict); + goto error; + } + return 0; + +error: + Py_DECREF(dict); + return -1; +} + +int +_PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, false) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, true) < 0) { + return -1; + } + return 0; +} From a1130179235e677cd9c84f4de092b32d1a05d260 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 11:41:37 -0600 Subject: [PATCH 06/19] _PyInterpreterState_ResolveConfig() -> _PyInterpreterConfig_InitFromState() --- Include/internal/pycore_pylifecycle.h | 12 ++++++------ Modules/_testinternalcapi.c | 6 +++--- Python/interpconfig.c | 24 +++++++++++++++++++++++- Python/pystate.c | 22 ---------------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index a230cd2c5b927a..47ff0806574ac0 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -120,13 +120,13 @@ int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCo /* interpreter config */ // Export for _testinternalcapi shared extension -PyAPI_FUNC(int) _PyInterpreterState_ResolveConfig( - PyInterpreterState *, - PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromState( + PyInterpreterConfig *, + PyInterpreterState *); PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); -PyAPI_FUNC(int) _PyInterpreterConfig_FromDict( - PyObject *, - PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromDict( + PyInterpreterConfig *, + PyObject *); PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( PyInterpreterConfig *, PyObject *); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index aff6af9e47c75d..05bf0947cf74eb 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -28,7 +28,7 @@ #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_pylifecycle.h" // _PyInterpreterState_ResolveConfig() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_InitFromState() #include "pycore_pystate.h" // _PyThreadState_GET() #include "clinic/_testinternalcapi.c.h" @@ -1399,7 +1399,7 @@ get_interpreter_config(PyObject *self, PyObject *args) } PyInterpreterConfig config; - if (_PyInterpreterState_ResolveConfig(interp, &config) < 0) { + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { return NULL; } PyObject *dict = _PyInterpreterConfig_AsDict(&config); @@ -1484,7 +1484,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyInterpreterConfig config = {0}; - int res = _PyInterpreterConfig_FromDict(dict, &config); + int res = _PyInterpreterConfig_InitFromDict(&config, dict); Py_DECREF(dict); if (res < 0) { return NULL; diff --git a/Python/interpconfig.c b/Python/interpconfig.c index 601fc6744567e8..e866326fdee07a 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -215,7 +215,7 @@ interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, } int -_PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) +_PyInterpreterConfig_InitFromDict(PyInterpreterConfig *config, PyObject *dict) { if (!PyDict_Check(dict)) { PyErr_SetString(PyExc_TypeError, "dict expected"); @@ -239,3 +239,25 @@ _PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) } return 0; } + +int +_PyInterpreterConfig_InitFromState(PyInterpreterConfig *config, + PyInterpreterState *interp) +{ + // Populate the config by re-constructing the values from the interpreter. + *config = (PyInterpreterConfig){ +#define FLAG(flag) \ + (interp->feature_flags & Py_RTFLAGS_ ## flag) + .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), + .allow_fork = FLAG(FORK), + .allow_exec = FLAG(EXEC), + .allow_threads = FLAG(THREADS), + .allow_daemon_threads = FLAG(DAEMON_THREADS), + .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), +#undef FLAG + .gil = interp->ceval.own_gil + ? PyInterpreterConfig_OWN_GIL + : PyInterpreterConfig_SHARED_GIL, + }; + return 0; +} diff --git a/Python/pystate.c b/Python/pystate.c index 4a2106f19701c8..47d327ae28933b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1122,28 +1122,6 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) return interp->dict; } -int -_PyInterpreterState_ResolveConfig(PyInterpreterState *interp, - PyInterpreterConfig *config) -{ - // Populate the config by re-constructing the values from the interpreter. - *config = (PyInterpreterConfig){ -#define FLAG(flag) \ - (interp->feature_flags & Py_RTFLAGS_ ## flag) - .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), - .allow_fork = FLAG(FORK), - .allow_exec = FLAG(EXEC), - .allow_threads = FLAG(THREADS), - .allow_daemon_threads = FLAG(DAEMON_THREADS), - .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), -#undef FLAG - .gil = interp->ceval.own_gil - ? PyInterpreterConfig_OWN_GIL - : PyInterpreterConfig_SHARED_GIL, - }; - return 0; -} - //---------- // interp ID From fce72b818ae9c4c264df5c1bb9b2cc6b0ff0efd0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 16:12:27 -0600 Subject: [PATCH 07/19] _testinternalcapi.new_interpreter_config() -> _xxsubinterpreters.new_config() --- Lib/test/test_interpreters/test_api.py | 143 ++++++++++++++++++++++++- Lib/test/test_interpreters/utils.py | 21 +++- Modules/_testinternalcapi.c | 53 --------- Modules/_xxsubinterpretersmodule.c | 76 +++++++++++++ Python/interpconfig.c | 17 +-- 5 files changed, 246 insertions(+), 64 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 3cde9bd0014d9a..73b28f0877d2e9 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,18 +1,27 @@ import os import pickle -import threading from textwrap import dedent +import threading +import types import unittest +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None from test import support from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +_interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError from .utils import _captured_script, _run_output, _running, TestBase +def requires__testinternalcapi(func): + return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) + + class ModuleTests(TestBase): def test_queue_aliases(self): @@ -932,6 +941,136 @@ class SubBytes(bytes): interpreters.is_shareable(obj)) +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + @requires__testinternalcapi + def test_new_config(self): + default = _interpreters.new_config('isolated') + with self.subTest('no arg'): + config = _interpreters.new_config() + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + with self.subTest('default'): + config1 = _interpreters.new_config('default') + self.assert_ns_equal(config1, default) + self.assertIsNot(config1, default) + + config2 = _interpreters.new_config('default') + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + for arg in ['', 'default']: + with self.subTest(f'default ({arg!r})'): + config = _interpreters.new_config(arg) + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=False, + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + for name, vanilla in supported.items(): + with self.subTest(f'supported ({name})'): + expected = vanilla + config1 = _interpreters.new_config(name) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(name) + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + with self.subTest(f'noop override ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'override all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('extra override'): + with self.assertRaises(ValueError): + _interpreters.new_config(spam=True) + + # Bad values for bool fields. + for field, value in vars(supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'bad override ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 973d05d4f96dcb..5ade6762ea24ef 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -68,6 +68,9 @@ def run(): class TestBase(unittest.TestCase): + def tearDown(self): + clean_up_interpreters() + def pipe(self): def ensure_closed(fd): try: @@ -156,5 +159,19 @@ def assert_python_failure(self, *argv): self.assertNotEqual(exitcode, 0) return stdout, stderr - def tearDown(self): - clean_up_interpreters() + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 05bf0947cf74eb..f7c44afacfa81f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1412,57 +1412,6 @@ get_interpreter_config(PyObject *self, PyObject *args) return configobj; } -static PyObject * -new_interpreter_config(PyObject *self, PyObject *args, PyObject *kwargs) -{ - const char *initialized = NULL; - PyObject *overrides = NULL; - static char *kwlist[] = {"initialized", "overrides", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|sO:new_interpreter_config", kwlist, - &initialized, &overrides)) - { - return NULL; - } - if (initialized == NULL - || strcmp(initialized, "") == 0 - || strcmp(initialized, "default") == 0) - { - initialized = "isolated"; - } - - PyInterpreterConfig config; - if (strcmp(initialized, "isolated") == 0) { - config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; - } - else if (strcmp(initialized, "legacy") == 0) { - config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - } - else if (strcmp(initialized, "empty") == 0) { - config = (PyInterpreterConfig){0}; - } - else { - PyErr_Format(PyExc_ValueError, - "unsupported initialized arg '%s'", initialized); - return NULL; - } - - if (overrides != NULL) { - if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { - return NULL; - } - } - - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - /* To run some code in a sub-interpreter. */ static PyObject * @@ -1903,8 +1852,6 @@ static PyMethodDef module_functions[] = { {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, {"get_interpreter_config", get_interpreter_config, METH_VARARGS}, - {"new_interpreter_config", _PyCFunction_CAST(new_interpreter_config), - METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index befa225c9183c5..a0dfdc7d52068a 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -12,8 +12,10 @@ #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -349,6 +351,34 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) /* interpreter-specific code ************************************************/ +static int +init_named_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL + || strcmp(name, "") == 0 + || strcmp(name, "default") == 0) + { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -417,6 +447,50 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ +static PyObject * +interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", + &name)) + { + return NULL; + } + PyObject *overrides = kwds; + + PyInterpreterConfig config; + if (init_named_config(&config, name) < 0) { + return NULL; + } + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(new_config_doc, +"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +\n\ +Return a representation of a new PyInterpreterConfig.\n\ +\n\ +The name determines the initial values of the config. Supported named\n\ +configs are: default, isolated, legacy, and empty.\n\ +\n\ +Any keyword arguments are set on the corresponding config fields,\n\ +overriding the initial values."); + + static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1033,6 +1107,8 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) static PyMethodDef module_functions[] = { + {"new_config", _PyCFunction_CAST(interp_new_config), + METH_VARARGS | METH_KEYWORDS, new_config_doc}, {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), diff --git a/Python/interpconfig.c b/Python/interpconfig.c index e866326fdee07a..a4eb9d7ee9aebc 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -29,7 +29,7 @@ static int gil_flag_from_str(const char *str, int *p_flag) { int flag; - if (str == NULL || strcmp(str, "") == 0) { + if (str == NULL) { flag = PyInterpreterConfig_DEFAULT_GIL; } else if (strcmp(str, "default") == 0) { @@ -42,8 +42,8 @@ gil_flag_from_str(const char *str, int *p_flag) flag = PyInterpreterConfig_OWN_GIL; } else { - PyErr_SetString(PyExc_SystemError, - "invalid interpreter config 'gil' value"); + PyErr_Format(PyExc_ValueError, + "unsupported interpreter config .gil value '%s'", str); return -1; } *p_flag = flag; @@ -73,7 +73,7 @@ _PyInterpreterConfig_AsDict(PyInterpreterConfig *config) if (STR == NULL) { \ goto error; \ } \ - PyObject *obj = PyUnicode_FromString(#FIELD); \ + PyObject *obj = PyUnicode_FromString(STR); \ if (obj == NULL) { \ goto error; \ } \ @@ -107,11 +107,14 @@ _config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) if (_config_dict_get(dict, name, &item) < 0) { return -1; } - int flag = PyObject_IsTrue(item); - Py_DECREF(item); - if (flag < 0) { + // For now we keep things strict, rather than using PyObject_IsTrue(). + int flag = item == Py_True; + if (!flag && item != Py_False) { + Py_DECREF(item); + config_dict_invalid_type(name); return -1; } + Py_DECREF(item); *p_flag = flag; return 0; } From c52e484172de0a7dd768f1db8402455d4578f524 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 15:40:56 -0600 Subject: [PATCH 08/19] _testinternalcapi.get_interpreter_config() -> _xxsubinterpreters.get_config() --- Lib/test/test_interpreters/test_api.py | 22 ++++++++++++- Modules/_testinternalcapi.c | 38 +--------------------- Modules/_xxsubinterpretersmodule.c | 44 ++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 73b28f0877d2e9..a584e0677c59e1 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -950,7 +950,6 @@ class LowLevelTests(TestBase): # encountered by the high-level module, thus they # mostly shouldn't matter as much. - @requires__testinternalcapi def test_new_config(self): default = _interpreters.new_config('isolated') with self.subTest('no arg'): @@ -1070,6 +1069,27 @@ def test_new_config(self): with self.assertRaises(ValueError): _interpreters.new_config(gil=value) + @requires__testinternalcapi + def test_get_config(self): + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + interpid = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create(isolated=True) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create(isolated=False) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f7c44afacfa81f..fe3c1241309dec 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,12 +23,11 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() -#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_pylifecycle.h" // _PyInterpreterConfig_InitFromState() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyThreadState_GET() #include "clinic/_testinternalcapi.c.h" @@ -1379,40 +1378,6 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -static PyObject * -get_interpreter_config(PyObject *self, PyObject *args) -{ - PyObject *idobj = NULL; - if (!PyArg_ParseTuple(args, "|O:get_interpreter_config", &idobj)) { - return NULL; - } - - PyInterpreterState *interp; - if (idobj == NULL) { - interp = PyInterpreterState_Get(); - } - else { - interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - } - - PyInterpreterConfig config; - if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { - return NULL; - } - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - - /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1851,7 +1816,6 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, - {"get_interpreter_config", get_interpreter_config, METH_VARARGS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index a0dfdc7d52068a..cd69701e0d6518 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1063,6 +1063,48 @@ PyDoc_STRVAR(is_running_doc, Return whether or not the identified interpreter is running."); +static PyObject * +interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:get_config", kwlist, &idobj)) + { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(get_config_doc, +"get_config(id) -> types.SimpleNamespace\n\ +\n\ +Return a representation of the config used to initialize the interpreter."); + + static PyObject * interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1122,6 +1164,8 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, + {"get_config", _PyCFunction_CAST(interp_get_config), + METH_VARARGS | METH_KEYWORDS, get_config_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, {"call", _PyCFunction_CAST(interp_call), From a2983ceda47c57cc5655625a35cddc7dcb3f630d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 14:53:58 -0600 Subject: [PATCH 09/19] Call _PyInterpreterState_RequireIDRef() in _interpreters._incref(). --- Modules/_xxsubinterpretersmodule.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index cd69701e0d6518..af0d944c78a271 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -540,7 +540,6 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) PyThreadState_Swap(save_tstate); PyThreadState_Delete(tstate); - _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } @@ -1108,10 +1107,13 @@ Return a representation of the config used to initialize the interpreter."); static PyObject * interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "implieslink", NULL}; PyObject *id; + int implieslink = -1; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O|$p:_incref", kwlist, + &id, &implieslink)) + { return NULL; } @@ -1119,6 +1121,14 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } + if (implieslink < 0) { + implieslink = !_Py_IsMainInterpreter(interp); + } + + if (implieslink) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } if (_PyInterpreterState_IDInitref(interp) < 0) { return NULL; } From 05a081edecc3085324d8139e1ad28d55247d7292 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 19:05:31 -0600 Subject: [PATCH 10/19] _testinternalcapi.interpreter_incref() -> _interpreters._incref() --- Lib/test/test_capi/test_misc.py | 18 +++++++++--------- Modules/_testinternalcapi.c | 24 ------------------------ 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 55a1ab6d6d9359..c8f606e58b13d1 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2284,8 +2284,8 @@ def test_linked_lifecycle_does_not_exist(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) + decref = _interpreters._decref with self.subTest('never existed'): interpid = _testinternalcapi.unused_interpreter_id() @@ -2339,8 +2339,8 @@ def test_linked_lifecycle_never_linked(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) + decref = _interpreters._decref interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) @@ -2389,8 +2389,8 @@ def test_linked_lifecycle_link_incref_decref(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) + decref = _interpreters._decref interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) @@ -2416,7 +2416,7 @@ def test_linked_lifecycle_incref_link(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) @@ -2438,8 +2438,8 @@ def test_linked_lifecycle_link_incref_unlink_decref(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) + decref = _interpreters._decref interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index fe3c1241309dec..8a2baa266b3d9e 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1557,28 +1557,6 @@ interpreter_refcount_linked(PyObject *self, PyObject *idobj) Py_RETURN_FALSE; } -static PyObject * -interpreter_incref(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - _PyInterpreterState_IDIncref(interp); - Py_RETURN_NONE; -} - -static PyObject * -interpreter_decref(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - _PyInterpreterState_IDDecref(interp); - Py_RETURN_NONE; -} - static void _xid_capsule_destructor(PyObject *capsule) @@ -1827,8 +1805,6 @@ static PyMethodDef module_functions[] = { {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, - {"interpreter_incref", interpreter_incref, METH_O}, - {"interpreter_decref", interpreter_decref, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, From 8a39bbc8a1b2add89d521367cd25da813c71498b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 14:22:12 -0600 Subject: [PATCH 11/19] Supporting passing a config to _xxsubinterpreters.create(). --- Lib/test/support/interpreters/__init__.py | 2 +- Lib/test/test__xxsubinterpreters.py | 4 +- Lib/test/test_importlib/test_util.py | 4 +- Lib/test/test_interpreters/test_api.py | 57 +++++++++++++++++++++-- Modules/_xxsubinterpretersmodule.c | 44 +++++++++++++---- 5 files changed, 95 insertions(+), 16 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index d8e6654fc96efd..f38ce1c9d84ae8 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -73,7 +73,7 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=True) + id = _interpreters.create() return Interpreter(id) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 35d7355680e549..f674771c27cbb1 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -584,7 +584,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create(isolated=True) + subinterp = interpreters.create('isolated') script, file = _captured_script(f""" import threading def f(): @@ -604,7 +604,7 @@ def f(): self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create(isolated=False) + subinterp = interpreters.create('legacy') script, file = _captured_script(""" import threading def f(): diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index a6a76e589761e0..115cb7a56c98f7 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -656,7 +656,7 @@ def test_magic_number(self): class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): - interpid = _interpreters.create(isolated=True) + interpid = _interpreters.create('isolated') def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -669,7 +669,7 @@ def ensure_destroyed(): raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index a584e0677c59e1..fed84a11f06084 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1069,7 +1069,6 @@ def test_new_config(self): with self.assertRaises(ValueError): _interpreters.new_config(gil=value) - @requires__testinternalcapi def test_get_config(self): with self.subTest('main'): expected = _interpreters.new_config('legacy') @@ -1080,16 +1079,68 @@ def test_get_config(self): with self.subTest('isolated'): expected = _interpreters.new_config('isolated') - interpid = _interpreters.create(isolated=True) + interpid = _interpreters.create('isolated') config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('legacy'): expected = _interpreters.new_config('legacy') - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) + @requires__testinternalcapi + def test_create(self): + isolated = _interpreters.new_config('isolated') + legacy = _interpreters.new_config('legacy') + default = isolated + + with self.subTest('no arg'): + interpid = _interpreters.create() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('arg: None'): + interpid = _interpreters.create(None) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('arg: \'empty\''): + with self.assertRaises(RuntimeError): + # The "empty" config isn't viable on its own. + _interpreters.create('empty') + + for arg, expected in { + '': default, + 'default': default, + 'isolated': isolated, + 'legacy': legacy, + }.items(): + with self.subTest(f'str arg: {arg!r}'): + interpid = _interpreters.create(arg) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config('empty') + orig.use_main_obmalloc = True + orig.gil = 'shared' + interpid = _interpreters.create(orig) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + with self.subTest('missing fields'): + orig = _interpreters.new_config() + del orig.gil + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('extra fields'): + orig = _interpreters.new_config() + orig.spam = True + with self.assertRaises(ValueError): + _interpreters.create(orig) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index af0d944c78a271..8d0fd72b63f489 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -378,6 +378,34 @@ init_named_config(PyInterpreterConfig *config, const char *name) return 0; } +static int +config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) @@ -494,21 +522,21 @@ overriding the initial values."); static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"config", NULL}; + PyObject *configobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:create", kwlist, + &configobj)) { + return NULL; + } - static char *kwlist[] = {"isolated", NULL}; - int isolated = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, - &isolated)) { + PyInterpreterConfig config; + if (config_from_object(configobj, &config) < 0) { return NULL; } // Create and initialize the new interpreter. PyThreadState *save_tstate = PyThreadState_Get(); assert(save_tstate != NULL); - const PyInterpreterConfig config = isolated - ? (PyInterpreterConfig)_PyInterpreterConfig_INIT - : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - // XXX Possible GILState issues? PyThreadState *tstate = NULL; PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); From 1173cd1cbb77b8abfe8d6d2079b9523788ec11c3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 14:52:57 -0600 Subject: [PATCH 12/19] Factor out new_interpreter(). --- Lib/test/test_capi/test_misc.py | 12 ++--- Modules/_testinternalcapi.c | 45 ---------------- Modules/_xxsubinterpretersmodule.c | 87 ++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 79 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index c8f606e58b13d1..4db2cbf7e9c2ff 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2327,7 +2327,7 @@ def test_linked_lifecycle_initial(self): get_refcount = _testinternalcapi.get_interpreter_refcount # A new interpreter will start out not linked, with a refcount of 0. - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) linked = is_linked(interpid) refcount = get_refcount(interpid) @@ -2342,7 +2342,7 @@ def test_linked_lifecycle_never_linked(self): incref = (lambda id: _interpreters._incref(id, implieslink=False)) decref = _interpreters._decref - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) # Incref will not automatically link it. @@ -2367,7 +2367,7 @@ def test_linked_lifecycle_link_unlink(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) # Linking at refcount 0 does not destroy the interpreter. @@ -2392,7 +2392,7 @@ def test_linked_lifecycle_link_incref_decref(self): incref = (lambda id: _interpreters._incref(id, implieslink=False)) decref = _interpreters._decref - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) # Linking it will not change the refcount. @@ -2418,7 +2418,7 @@ def test_linked_lifecycle_incref_link(self): get_refcount = _testinternalcapi.get_interpreter_refcount incref = (lambda id: _interpreters._incref(id, implieslink=False)) - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) incref(interpid) @@ -2441,7 +2441,7 @@ def test_linked_lifecycle_link_incref_unlink_decref(self): incref = (lambda id: _interpreters._incref(id, implieslink=False)) decref = _interpreters._decref - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) link(interpid) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 8a2baa266b3d9e..daac1b45e66589 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1451,50 +1451,6 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromLongLong(interpid); } -static PyObject * -new_interpreter(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - // Unlike _interpreters.create(), we do not automatically link - // the interpreter to its refcount. - PyThreadState *save_tstate = PyThreadState_Get(); - const PyInterpreterConfig config = \ - (PyInterpreterConfig)_PyInterpreterConfig_INIT; - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - _PyErr_SetFromPyStatus(status); - return NULL; - } - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - - if (_PyInterpreterState_IDInitref(interp) < 0) { - goto error; - } - - int64_t interpid = PyInterpreterState_GetID(interp); - if (interpid < 0) { - goto error; - } - PyObject *idobj = PyLong_FromLongLong(interpid); - if (idobj == NULL) { - goto error; - } - - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - - return idobj; - -error: - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; -} - static PyObject * interpreter_exists(PyObject *self, PyObject *idobj) { @@ -1799,7 +1755,6 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, - {"new_interpreter", new_interpreter, METH_NOARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 8d0fd72b63f489..6cd3045711e31f 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -407,6 +407,59 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } +static PyInterpreterState * +new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate) +{ + PyThreadState *save_tstate = PyThreadState_Get(); + assert(save_tstate != NULL); + PyThreadState *tstate = NULL; + // XXX Possible GILState issues? + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + _PyErr_SetFromPyStatus(status); + return NULL; + } + assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + if (_PyInterpreterState_IDInitref(interp) < 0) { + goto error; + } + + if (p_idobj != NULL) { + // We create the object using the original interpreter. + PyObject *idobj = get_interpid_obj(interp); + if (idobj == NULL) { + goto error; + } + *p_idobj = idobj; + } + + if (p_tstate != NULL) { + *p_tstate = tstate; + } + else { + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + } + + return interp; + +error: + // XXX Possible GILState issues? + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; +} + + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -534,43 +587,21 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - // XXX Possible GILState issues? - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); + PyObject *idobj = NULL; + PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL); + if (interp == NULL) { + // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } - assert(tstate != NULL); - - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = get_interpid_obj(interp); - if (idobj == NULL) { - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; - } - - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); return idobj; } + PyDoc_STRVAR(create_doc, "create() -> ID\n\ \n\ From 92c11d388baf7648f781937405c790370bc74674 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 25 Mar 2024 09:55:07 -0600 Subject: [PATCH 13/19] Fix test_import. --- Lib/test/test_import/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 81ec700d9755ce..3c387d973ce0f9 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2163,7 +2163,7 @@ def re_load(self, name, mod): # subinterpreters def add_subinterpreter(self): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) From edda48dbe64ccc4d5e67f8eb60d0399935f59ea8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 25 Mar 2024 13:49:35 -0600 Subject: [PATCH 14/19] Fix an outdent. --- Python/interpconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/interpconfig.c b/Python/interpconfig.c index a4eb9d7ee9aebc..419f40ae62a89e 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -73,7 +73,7 @@ _PyInterpreterConfig_AsDict(PyInterpreterConfig *config) if (STR == NULL) { \ goto error; \ } \ - PyObject *obj = PyUnicode_FromString(STR); \ + PyObject *obj = PyUnicode_FromString(STR); \ if (obj == NULL) { \ goto error; \ } \ From 5f617ed2ed7a4b5dfb451e5f3846fb72951cb116 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 1 Apr 2024 11:56:16 -0600 Subject: [PATCH 15/19] Call _PyInterpreterState_RequireIDRef() in the right places. --- Lib/test/support/interpreters/__init__.py | 2 +- Modules/_xxsubinterpretersmodule.c | 28 +++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index f38ce1c9d84ae8..784724c1b94a6c 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -73,7 +73,7 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create() + id = _interpreters.create(reqrefs=True) return Interpreter(id) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 6cd3045711e31f..07d6e692bdb08b 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -575,10 +575,11 @@ overriding the initial values."); static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"config", NULL}; + static char *kwlist[] = {"config", "reqrefs", NULL}; PyObject *configobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:create", kwlist, - &configobj)) { + int reqrefs = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, + &configobj, &reqrefs)) { return NULL; } @@ -598,16 +599,28 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } + if (reqrefs) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } + return idobj; } PyDoc_STRVAR(create_doc, -"create() -> ID\n\ +"create([config], *, reqrefs=False) -> ID\n\ \n\ Create a new interpreter and return a unique generated ID.\n\ \n\ -The caller is responsible for destroying the interpreter before exiting."); +The caller is responsible for destroying the interpreter before exiting,\n\ +typically by using _interpreters.destroy(). This can be managed \n\ +automatically by passing \"reqrefs=True\" and then using _incref() and\n\ +_decref()` appropriately.\n\ +\n\ +\"config\" must be a valid interpreter config or the name of a\n\ +predefined config (\"isolated\" or \"legacy\"). The default\n\ +is \"isolated\"."); static PyObject * @@ -1168,7 +1181,7 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", "implieslink", NULL}; PyObject *id; - int implieslink = -1; + int implieslink = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$p:_incref", kwlist, &id, &implieslink)) @@ -1180,9 +1193,6 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - if (implieslink < 0) { - implieslink = !_Py_IsMainInterpreter(interp); - } if (implieslink) { // Decref to 0 will destroy the interpreter. From c504c79e3eccf8dac373fbf489c1da058691ef11 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 1 Apr 2024 11:57:23 -0600 Subject: [PATCH 16/19] Drop an unnecessary _PyInterpreterState_IDInitref() call. --- Modules/_xxsubinterpretersmodule.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 07d6e692bdb08b..f1a61f66abb4e7 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1198,9 +1198,6 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) // Decref to 0 will destroy the interpreter. _PyInterpreterState_RequireIDRef(interp, 1); } - if (_PyInterpreterState_IDInitref(interp) < 0) { - return NULL; - } _PyInterpreterState_IDIncref(interp); Py_RETURN_NONE; From 8a75c9002b0a1f52b64bb7ded652a4a6e48f4e58 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 13:39:58 -0600 Subject: [PATCH 17/19] Reduce to just the new internal C-API. --- Lib/test/support/interpreters/__init__.py | 2 +- Lib/test/test__xxsubinterpreters.py | 4 +- Lib/test/test_capi/test_misc.py | 280 ++++++++++++++++++++-- Lib/test/test_import/__init__.py | 2 +- Lib/test/test_importlib/test_util.py | 4 +- Lib/test/test_interpreters/test_api.py | 214 +---------------- Lib/test/test_interpreters/test_queues.py | 6 +- Lib/test/test_interpreters/utils.py | 21 +- Modules/_testinternalcapi.c | 217 ++++++++++++++++- Modules/_xxsubinterpretersmodule.c | 278 ++++----------------- 10 files changed, 526 insertions(+), 502 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 784724c1b94a6c..d8e6654fc96efd 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -73,7 +73,7 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(reqrefs=True) + id = _interpreters.create(isolated=True) return Interpreter(id) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index f674771c27cbb1..35d7355680e549 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -584,7 +584,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create('isolated') + subinterp = interpreters.create(isolated=True) script, file = _captured_script(f""" import threading def f(): @@ -604,7 +604,7 @@ def f(): self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create('legacy') + subinterp = interpreters.create(isolated=False) script, file = _captured_script(""" import threading def f(): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 4db2cbf7e9c2ff..a50f9c0f0dc822 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2204,6 +2204,256 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +class InterpreterConfigTests(unittest.TestCase): + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=False, + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + def iter_all_configs(self): + for use_main_obmalloc in (True, False): + for allow_fork in (True, False): + for allow_exec in (True, False): + for allow_threads in (True, False): + for allow_daemon in (True, False): + for checkext in (True, False): + for gil in ('shared', 'own', 'default'): + yield types.SimpleNamespace( + use_main_obmalloc=use_main_obmalloc, + allow_fork=allow_fork, + allow_exec=allow_exec, + allow_threads=allow_threads, + allow_daemon_threads=allow_daemon, + check_multi_interp_extensions=checkext, + gil=gil, + ) + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def test_predefined_config(self): + def check(name, expected): + expected = self.supported[expected] + args = (name,) if name else () + + config1 = _testinternalcapi.new_interp_config(*args) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _testinternalcapi.new_interp_config(*args) + self.assert_ns_equal(config2, expected) + self.assertIsNot(config2, expected) + self.assertIsNot(config2, config1) + + with self.subTest('default'): + check(None, 'isolated') + + for name in self.supported: + with self.subTest(name): + check(name, name) + + def test_update_from_dict(self): + for name, vanilla in self.supported.items(): + with self.subTest(f'noop ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _testinternalcapi.new_interp_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'change all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in self.gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _testinternalcapi.new_interp_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in self.gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _testinternalcapi.new_interp_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('unsupported field'): + for name in self.supported: + with self.assertRaises(ValueError): + _testinternalcapi.new_interp_config(name, spam=True) + + # Bad values for bool fields. + for field, value in vars(self.supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'unsupported value ({field}={value!r})'): + with self.assertRaises(TypeError): + _testinternalcapi.new_interp_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'unsupported value(gil={value!r})'): + with self.assertRaises(TypeError): + _testinternalcapi.new_interp_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'unsupported value (gil={value!r})'): + with self.assertRaises(ValueError): + _testinternalcapi.new_interp_config(gil=value) + + @requires_subinterpreters + def test_interp_init(self): + questionable = [ + # strange + dict( + allow_fork=True, + allow_exec=False, + ), + dict( + gil='shared', + use_main_obmalloc=False, + ), + # risky + dict( + allow_fork=True, + allow_threads=True, + ), + # ought to be invalid? + dict( + allow_threads=False, + allow_daemon_threads=True, + ), + dict( + gil='own', + use_main_obmalloc=True, + ), + ] + invalid = [ + dict( + use_main_obmalloc=False, + check_multi_interp_extensions=False + ), + ] + def match(config, override_cases): + ns = vars(config) + for overrides in override_cases: + if dict(ns, **overrides) == ns: + return True + return False + + def check(config): + script = 'pass' + rc = _testinternalcapi.run_in_subinterp_with_config(script, config) + self.assertEqual(rc, 0) + + for config in self.iter_all_configs(): + if config.gil == 'default': + continue + if match(config, invalid): + with self.subTest(f'invalid: {config}'): + with self.assertRaises(RuntimeError): + check(config) + elif match(config, questionable): + with self.subTest(f'questionable: {config}'): + check(config) + else: + with self.subTest(f'valid: {config}'): + check(config) + + @requires_subinterpreters + def test_get_config(self): + def new_interp(config): + interpid = _testinternalcapi.new_interpreter(config) + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + return interpid + + with self.subTest('main'): + expected = _testinternalcapi.new_interp_config('legacy') + expected.gil = 'own' + interpid = _interpreters.get_main() + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _testinternalcapi.new_interp_config('isolated') + interpid = new_interp('isolated') + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _testinternalcapi.new_interp_config('legacy') + interpid = new_interp('legacy') + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _testinternalcapi.new_interp_config( + 'empty', + use_main_obmalloc=True, + gil='shared', + ) + interpid = new_interp(orig) + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, orig) + + @requires_subinterpreters class InterpreterIDTests(unittest.TestCase): @@ -2284,8 +2534,8 @@ def test_linked_lifecycle_does_not_exist(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) - decref = _interpreters._decref + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref with self.subTest('never existed'): interpid = _testinternalcapi.unused_interpreter_id() @@ -2327,7 +2577,7 @@ def test_linked_lifecycle_initial(self): get_refcount = _testinternalcapi.get_interpreter_refcount # A new interpreter will start out not linked, with a refcount of 0. - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) linked = is_linked(interpid) refcount = get_refcount(interpid) @@ -2339,10 +2589,10 @@ def test_linked_lifecycle_never_linked(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) - decref = _interpreters._decref + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) # Incref will not automatically link it. @@ -2367,7 +2617,7 @@ def test_linked_lifecycle_link_unlink(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) # Linking at refcount 0 does not destroy the interpreter. @@ -2389,10 +2639,10 @@ def test_linked_lifecycle_link_incref_decref(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) - decref = _interpreters._decref + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) # Linking it will not change the refcount. @@ -2416,9 +2666,9 @@ def test_linked_lifecycle_incref_link(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) + incref = _testinternalcapi.interpreter_incref - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) incref(interpid) @@ -2438,10 +2688,10 @@ def test_linked_lifecycle_link_incref_unlink_decref(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) - decref = _interpreters._decref + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) link(interpid) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 3c387d973ce0f9..81ec700d9755ce 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2163,7 +2163,7 @@ def re_load(self, name, mod): # subinterpreters def add_subinterpreter(self): - interpid = _interpreters.create('legacy') + interpid = _interpreters.create(isolated=False) def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 115cb7a56c98f7..a6a76e589761e0 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -656,7 +656,7 @@ def test_magic_number(self): class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): - interpid = _interpreters.create('isolated') + interpid = _interpreters.create(isolated=True) def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -669,7 +669,7 @@ def ensure_destroyed(): raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create('legacy') + interpid = _interpreters.create(isolated=False) def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index fed84a11f06084..3cde9bd0014d9a 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,27 +1,18 @@ import os import pickle -from textwrap import dedent import threading -import types +from textwrap import dedent import unittest -try: - import _testinternalcapi -except ImportError: - _testinternalcapi = None from test import support from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -_interpreters = import_helper.import_module('_xxsubinterpreters') +import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError from .utils import _captured_script, _run_output, _running, TestBase -def requires__testinternalcapi(func): - return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) - - class ModuleTests(TestBase): def test_queue_aliases(self): @@ -941,207 +932,6 @@ class SubBytes(bytes): interpreters.is_shareable(obj)) -class LowLevelTests(TestBase): - - # The behaviors in the low-level module are important in as much - # as they are exercised by the high-level module. Therefore the - # most important testing happens in the high-level tests. - # These low-level tests cover corner cases that are not - # encountered by the high-level module, thus they - # mostly shouldn't matter as much. - - def test_new_config(self): - default = _interpreters.new_config('isolated') - with self.subTest('no arg'): - config = _interpreters.new_config() - self.assert_ns_equal(config, default) - self.assertIsNot(config, default) - - with self.subTest('default'): - config1 = _interpreters.new_config('default') - self.assert_ns_equal(config1, default) - self.assertIsNot(config1, default) - - config2 = _interpreters.new_config('default') - self.assert_ns_equal(config2, config1) - self.assertIsNot(config2, config1) - - for arg in ['', 'default']: - with self.subTest(f'default ({arg!r})'): - config = _interpreters.new_config(arg) - self.assert_ns_equal(config, default) - self.assertIsNot(config, default) - - supported = { - 'isolated': types.SimpleNamespace( - use_main_obmalloc=False, - allow_fork=False, - allow_exec=False, - allow_threads=True, - allow_daemon_threads=False, - check_multi_interp_extensions=True, - gil='own', - ), - 'legacy': types.SimpleNamespace( - use_main_obmalloc=True, - allow_fork=True, - allow_exec=True, - allow_threads=True, - allow_daemon_threads=True, - check_multi_interp_extensions=False, - gil='shared', - ), - 'empty': types.SimpleNamespace( - use_main_obmalloc=False, - allow_fork=False, - allow_exec=False, - allow_threads=False, - allow_daemon_threads=False, - check_multi_interp_extensions=False, - gil='default', - ), - } - gil_supported = ['default', 'shared', 'own'] - - for name, vanilla in supported.items(): - with self.subTest(f'supported ({name})'): - expected = vanilla - config1 = _interpreters.new_config(name) - self.assert_ns_equal(config1, expected) - self.assertIsNot(config1, expected) - - config2 = _interpreters.new_config(name) - self.assert_ns_equal(config2, config1) - self.assertIsNot(config2, config1) - - with self.subTest(f'noop override ({name})'): - expected = vanilla - overrides = vars(vanilla) - config = _interpreters.new_config(name, **overrides) - self.assert_ns_equal(config, expected) - - with self.subTest(f'override all ({name})'): - overrides = {k: not v for k, v in vars(vanilla).items()} - for gil in gil_supported: - if vanilla.gil == gil: - continue - overrides['gil'] = gil - expected = types.SimpleNamespace(**overrides) - config = _interpreters.new_config(name, **overrides) - self.assert_ns_equal(config, expected) - - # Override individual fields. - for field, old in vars(vanilla).items(): - if field == 'gil': - values = [v for v in gil_supported if v != old] - else: - values = [not old] - for val in values: - with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): - overrides = {field: val} - expected = types.SimpleNamespace( - **dict(vars(vanilla), **overrides), - ) - config = _interpreters.new_config(name, **overrides) - self.assert_ns_equal(config, expected) - - with self.subTest('extra override'): - with self.assertRaises(ValueError): - _interpreters.new_config(spam=True) - - # Bad values for bool fields. - for field, value in vars(supported['empty']).items(): - if field == 'gil': - continue - assert isinstance(value, bool) - for value in [1, '', 'spam', 1.0, None, object()]: - with self.subTest(f'bad override ({field}={value!r})'): - with self.assertRaises(TypeError): - _interpreters.new_config(**{field: value}) - - # Bad values for .gil. - for value in [True, 1, 1.0, None, object()]: - with self.subTest(f'bad override (gil={value!r})'): - with self.assertRaises(TypeError): - _interpreters.new_config(gil=value) - for value in ['', 'spam']: - with self.subTest(f'bad override (gil={value!r})'): - with self.assertRaises(ValueError): - _interpreters.new_config(gil=value) - - def test_get_config(self): - with self.subTest('main'): - expected = _interpreters.new_config('legacy') - expected.gil = 'own' - interpid = _interpreters.get_main() - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('isolated'): - expected = _interpreters.new_config('isolated') - interpid = _interpreters.create('isolated') - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('legacy'): - expected = _interpreters.new_config('legacy') - interpid = _interpreters.create('legacy') - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - @requires__testinternalcapi - def test_create(self): - isolated = _interpreters.new_config('isolated') - legacy = _interpreters.new_config('legacy') - default = isolated - - with self.subTest('no arg'): - interpid = _interpreters.create() - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, default) - - with self.subTest('arg: None'): - interpid = _interpreters.create(None) - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, default) - - with self.subTest('arg: \'empty\''): - with self.assertRaises(RuntimeError): - # The "empty" config isn't viable on its own. - _interpreters.create('empty') - - for arg, expected in { - '': default, - 'default': default, - 'isolated': isolated, - 'legacy': legacy, - }.items(): - with self.subTest(f'str arg: {arg!r}'): - interpid = _interpreters.create(arg) - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('custom'): - orig = _interpreters.new_config('empty') - orig.use_main_obmalloc = True - orig.gil = 'shared' - interpid = _interpreters.create(orig) - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, orig) - - with self.subTest('missing fields'): - orig = _interpreters.new_config() - del orig.gil - with self.assertRaises(ValueError): - _interpreters.create(orig) - - with self.subTest('extra fields'): - orig = _interpreters.new_config() - orig.spam = True - with self.assertRaises(ValueError): - _interpreters.create(orig) - - if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 8ab9ebb354712a..d16d294b82d044 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -28,9 +28,9 @@ def tearDown(self): class LowLevelTests(TestBase): - # The behaviors in the low-level module are important in as much - # as they are exercised by the high-level module. Therefore the - # most important testing happens in the high-level tests. + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. # These low-level tests cover corner cases that are not # encountered by the high-level module, thus they # mostly shouldn't matter as much. diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 5ade6762ea24ef..973d05d4f96dcb 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -68,9 +68,6 @@ def run(): class TestBase(unittest.TestCase): - def tearDown(self): - clean_up_interpreters() - def pipe(self): def ensure_closed(fd): try: @@ -159,19 +156,5 @@ def assert_python_failure(self, *argv): self.assertNotEqual(exitcode, 0) return stdout, stderr - def assert_ns_equal(self, ns1, ns2, msg=None): - # This is mostly copied from TestCase.assertDictEqual. - self.assertEqual(type(ns1), type(ns2)) - if ns1 == ns2: - return - - import difflib - import pprint - from unittest.util import _common_shorten_repr - standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) - diff = ('\n' + '\n'.join(difflib.ndiff( - pprint.pformat(vars(ns1)).splitlines(), - pprint.pformat(vars(ns2)).splitlines()))) - diff = f'namespace({diff})' - standardMsg = self._truncateMessage(standardMsg, diff) - self.fail(self._formatMessage(msg, standardMsg)) + def tearDown(self): + clean_up_interpreters() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index daac1b45e66589..685193194ff9b6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,6 +23,7 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() @@ -830,7 +831,6 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } -// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -1378,6 +1378,129 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } +static int +init_named_interp_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL) { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static PyObject * +new_interp_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:new_config", &name)) { + return NULL; + } + PyObject *overrides = kwds; + + if (name == NULL) { + name = "isolated"; + } + + PyInterpreterConfig config; + if (init_named_interp_config(&config, name) < 0) { + return NULL; + } + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static PyObject * +get_interp_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:get_config", kwlist, &idobj)) + { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static int +interp_config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_interp_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_interp_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1392,15 +1515,8 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } - PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); - if (dict == NULL) { - PyErr_Format(PyExc_TypeError, "bad config %R", configobj); - return NULL; - } - PyInterpreterConfig config = {0}; - int res = _PyInterpreterConfig_InitFromDict(&config, dict); - Py_DECREF(dict); - if (res < 0) { + PyInterpreterConfig config; + if (interp_config_from_object(configobj, &config) < 0) { return NULL; } @@ -1451,6 +1567,58 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromLongLong(interpid); } +static PyObject * +new_interpreter(PyObject *self, PyObject *args) +{ + PyObject *configobj = NULL; + if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) { + return NULL; + } + + PyInterpreterConfig config; + if (interp_config_from_object(configobj, &config) < 0) { + return NULL; + } + + // Unlike _interpreters.create(), we do not automatically link + // the interpreter to its refcount. + PyThreadState *save_tstate = PyThreadState_Get(); + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + _PyErr_SetFromPyStatus(status); + return NULL; + } + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + if (_PyInterpreterState_IDInitref(interp) < 0) { + goto error; + } + + int64_t interpid = PyInterpreterState_GetID(interp); + if (interpid < 0) { + goto error; + } + PyObject *idobj = PyLong_FromLongLong(interpid); + if (idobj == NULL) { + goto error; + } + + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + + return idobj; + +error: + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; +} + static PyObject * interpreter_exists(PyObject *self, PyObject *idobj) { @@ -1513,6 +1681,28 @@ interpreter_refcount_linked(PyObject *self, PyObject *idobj) Py_RETURN_FALSE; } +static PyObject * +interpreter_incref(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDIncref(interp); + Py_RETURN_NONE; +} + +static PyObject * +interpreter_decref(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDDecref(interp); + Py_RETURN_NONE; +} + static void _xid_capsule_destructor(PyObject *capsule) @@ -1750,16 +1940,23 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"new_interp_config", _PyCFunction_CAST(new_interp_config), + METH_VARARGS | METH_KEYWORDS}, + {"get_interp_config", _PyCFunction_CAST(get_interp_config), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, + {"new_interpreter", new_interpreter, METH_VARARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, + {"interpreter_incref", interpreter_incref, METH_O}, + {"interpreter_decref", interpreter_decref, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index f1a61f66abb4e7..befa225c9183c5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -12,10 +12,8 @@ #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() -#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo -#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -351,115 +349,6 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) /* interpreter-specific code ************************************************/ -static int -init_named_config(PyInterpreterConfig *config, const char *name) -{ - if (name == NULL - || strcmp(name, "") == 0 - || strcmp(name, "default") == 0) - { - name = "isolated"; - } - - if (strcmp(name, "isolated") == 0) { - *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; - } - else if (strcmp(name, "legacy") == 0) { - *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - } - else if (strcmp(name, "empty") == 0) { - *config = (PyInterpreterConfig){0}; - } - else { - PyErr_Format(PyExc_ValueError, - "unsupported config name '%s'", name); - return -1; - } - return 0; -} - -static int -config_from_object(PyObject *configobj, PyInterpreterConfig *config) -{ - if (configobj == NULL || configobj == Py_None) { - if (init_named_config(config, NULL) < 0) { - return -1; - } - } - else if (PyUnicode_Check(configobj)) { - if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { - return -1; - } - } - else { - PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); - if (dict == NULL) { - PyErr_Format(PyExc_TypeError, "bad config %R", configobj); - return -1; - } - int res = _PyInterpreterConfig_InitFromDict(config, dict); - Py_DECREF(dict); - if (res < 0) { - return -1; - } - } - return 0; -} - - -static PyInterpreterState * -new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate) -{ - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - PyThreadState *tstate = NULL; - // XXX Possible GILState issues? - PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); - return NULL; - } - assert(tstate != NULL); - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - - if (_PyInterpreterState_IDInitref(interp) < 0) { - goto error; - } - - if (p_idobj != NULL) { - // We create the object using the original interpreter. - PyObject *idobj = get_interpid_obj(interp); - if (idobj == NULL) { - goto error; - } - *p_idobj = idobj; - } - - if (p_tstate != NULL) { - *p_tstate = tstate; - } - else { - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - } - - return interp; - -error: - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; -} - - static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -528,99 +417,65 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ -static PyObject * -interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) -{ - const char *name = NULL; - if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", - &name)) - { - return NULL; - } - PyObject *overrides = kwds; - - PyInterpreterConfig config; - if (init_named_config(&config, name) < 0) { - return NULL; - } - - if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { - if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { - return NULL; - } - } - - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - -PyDoc_STRVAR(new_config_doc, -"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ -\n\ -Return a representation of a new PyInterpreterConfig.\n\ -\n\ -The name determines the initial values of the config. Supported named\n\ -configs are: default, isolated, legacy, and empty.\n\ -\n\ -Any keyword arguments are set on the corresponding config fields,\n\ -overriding the initial values."); - - static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"config", "reqrefs", NULL}; - PyObject *configobj = NULL; - int reqrefs = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, - &configobj, &reqrefs)) { - return NULL; - } - PyInterpreterConfig config; - if (config_from_object(configobj, &config) < 0) { + static char *kwlist[] = {"isolated", NULL}; + int isolated = 1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, + &isolated)) { return NULL; } - PyObject *idobj = NULL; - PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL); - if (interp == NULL) { - // XXX Move the chained exception to interpreters.create()? + // Create and initialize the new interpreter. + PyThreadState *save_tstate = PyThreadState_Get(); + assert(save_tstate != NULL); + const PyInterpreterConfig config = isolated + ? (PyInterpreterConfig)_PyInterpreterConfig_INIT + : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + + // XXX Possible GILState issues? + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + _PyErr_SetFromPyStatus(status); PyObject *exc = PyErr_GetRaisedException(); - assert(exc != NULL); PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } + assert(tstate != NULL); - if (reqrefs) { - // Decref to 0 will destroy the interpreter. - _PyInterpreterState_RequireIDRef(interp, 1); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + PyObject *idobj = get_interpid_obj(interp); + if (idobj == NULL) { + // XXX Possible GILState issues? + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; } + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + + _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } - PyDoc_STRVAR(create_doc, -"create([config], *, reqrefs=False) -> ID\n\ +"create() -> ID\n\ \n\ Create a new interpreter and return a unique generated ID.\n\ \n\ -The caller is responsible for destroying the interpreter before exiting,\n\ -typically by using _interpreters.destroy(). This can be managed \n\ -automatically by passing \"reqrefs=True\" and then using _incref() and\n\ -_decref()` appropriately.\n\ -\n\ -\"config\" must be a valid interpreter config or the name of a\n\ -predefined config (\"isolated\" or \"legacy\"). The default\n\ -is \"isolated\"."); +The caller is responsible for destroying the interpreter before exiting."); static PyObject * @@ -1134,58 +989,13 @@ PyDoc_STRVAR(is_running_doc, Return whether or not the identified interpreter is running."); -static PyObject * -interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", NULL}; - PyObject *idobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:get_config", kwlist, &idobj)) - { - return NULL; - } - - PyInterpreterState *interp; - if (idobj == NULL) { - interp = PyInterpreterState_Get(); - } - else { - interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - } - - PyInterpreterConfig config; - if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { - return NULL; - } - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - -PyDoc_STRVAR(get_config_doc, -"get_config(id) -> types.SimpleNamespace\n\ -\n\ -Return a representation of the config used to initialize the interpreter."); - - static PyObject * interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "implieslink", NULL}; + static char *kwlist[] = {"id", NULL}; PyObject *id; - int implieslink = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:_incref", kwlist, - &id, &implieslink)) - { + "O:_incref", kwlist, &id)) { return NULL; } @@ -1193,10 +1003,8 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - - if (implieslink) { - // Decref to 0 will destroy the interpreter. - _PyInterpreterState_RequireIDRef(interp, 1); + if (_PyInterpreterState_IDInitref(interp) < 0) { + return NULL; } _PyInterpreterState_IDIncref(interp); @@ -1225,8 +1033,6 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) static PyMethodDef module_functions[] = { - {"new_config", _PyCFunction_CAST(interp_new_config), - METH_VARARGS | METH_KEYWORDS, new_config_doc}, {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), @@ -1240,8 +1046,6 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, - {"get_config", _PyCFunction_CAST(interp_get_config), - METH_VARARGS | METH_KEYWORDS, get_config_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, {"call", _PyCFunction_CAST(interp_call), From a38cda7b9a3b7889404b1299ff5db6126260cd25 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 14:00:45 -0600 Subject: [PATCH 18/19] Adjust test_get_config. --- Lib/test/test_capi/test_misc.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index a50f9c0f0dc822..34311afc93fc29 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2414,15 +2414,16 @@ def check(config): @requires_subinterpreters def test_get_config(self): + @contextlib.contextmanager def new_interp(config): interpid = _testinternalcapi.new_interpreter(config) - def ensure_destroyed(): + try: + yield interpid + finally: try: _interpreters.destroy(interpid) except _interpreters.InterpreterNotFoundError: pass - self.addCleanup(ensure_destroyed) - return interpid with self.subTest('main'): expected = _testinternalcapi.new_interp_config('legacy') @@ -2433,14 +2434,14 @@ def ensure_destroyed(): with self.subTest('isolated'): expected = _testinternalcapi.new_interp_config('isolated') - interpid = new_interp('isolated') - config = _testinternalcapi.get_interp_config(interpid) + with new_interp('isolated') as interpid: + config = _testinternalcapi.get_interp_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('legacy'): expected = _testinternalcapi.new_interp_config('legacy') - interpid = new_interp('legacy') - config = _testinternalcapi.get_interp_config(interpid) + with new_interp('legacy') as interpid: + config = _testinternalcapi.get_interp_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('custom'): @@ -2449,8 +2450,8 @@ def ensure_destroyed(): use_main_obmalloc=True, gil='shared', ) - interpid = new_interp(orig) - config = _testinternalcapi.get_interp_config(interpid) + with new_interp(orig) as interpid: + config = _testinternalcapi.get_interp_config(interpid) self.assert_ns_equal(config, orig) From cae0482fb30925440202068823703ac78efc2974 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 14:03:04 -0600 Subject: [PATCH 19/19] Remove trailing whitespace. --- Modules/_testinternalcapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 685193194ff9b6..a33c8aa4f57023 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1574,7 +1574,7 @@ new_interpreter(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) { return NULL; } - + PyInterpreterConfig config; if (interp_config_from_object(configobj, &config) < 0) { return NULL; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy