From 76a7d7bfe2572de83a8bc38653573fae210eef57 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2023 14:45:35 -0700 Subject: [PATCH 01/26] Make Interpreter() idempotent. --- Lib/test/support/interpreters.py | 27 ++++-- Lib/test/test_interpreters.py | 161 ++++++++++++++++++++++--------- 2 files changed, 130 insertions(+), 58 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index 089fe7ef56df78..f82581146d5c26 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -1,6 +1,7 @@ """Subinterpreters High Level Module.""" import time +import weakref import _xxsubinterpreters as _interpreters import _xxinterpchannels as _channels @@ -58,14 +59,26 @@ def get_main(): return Interpreter(id) +_known = weakref.WeakValueDictionary() + class Interpreter: """A single Python interpreter.""" - def __init__(self, id, *, isolated=None): - if not isinstance(id, (int, _interpreters.InterpreterID)): + def __new__(cls, id, /, *, isolated=None): + # There is only one instance for any given ID. + if isinstance(id, int): + id = _interpreters.InterpreterID(id, force=False) + elif not isinstance(id, _interpreters.InterpreterID): raise TypeError(f'id must be an int, got {id!r}') - self._id = id - self._isolated = isolated + key = int(id) + try: + self = _known[key] + except KeyError: + self = super().__new__(cls) + self._id = id + self._isolated = bool(isolated) + _known[key] = self + return self def __repr__(self): data = dict(id=int(self._id), isolated=self._isolated) @@ -75,12 +88,6 @@ def __repr__(self): def __hash__(self): return hash(self._id) - def __eq__(self, other): - if not isinstance(other, Interpreter): - return NotImplemented - else: - return other._id == self._id - @property def id(self): return self._id diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 5663706c0ccfb7..e3158b690de868 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -143,6 +143,23 @@ def test_after_destroy_some(self): self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) +class GetMainTests(TestBase): + + def test_id(self): + main = interpreters.get_main() + self.assertEqual(main.id, 0) + + def test_current(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertIs(main, current) + + def test_idempotent(self): + main1 = interpreters.get_main() + main2 = interpreters.get_main() + self.assertIs(main1, main2) + + class GetCurrentTests(TestBase): def test_main(self): @@ -151,7 +168,7 @@ def test_main(self): self.assertEqual(current, main) def test_subinterpreter(self): - main = _interpreters.get_main() + main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" from test.support import interpreters @@ -159,8 +176,38 @@ def test_subinterpreter(self): print(cur.id) """)) current = interpreters.Interpreter(int(out)) + self.assertEqual(current, interp) self.assertNotEqual(current, main) + def test_idempotent(self): + with self.subTest('main'): + cur1 = interpreters.get_current() + cur2 = interpreters.get_current() + self.assertIs(cur1, cur2) + + with self.subTest('subinterpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + cur = interpreters.get_current() + print(id(cur)) + """)) + objid1, objid2 = (int(v) for v in out.splitlines()) + self.assertEqual(objid1, objid2) + + with self.subTest('per-interpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + """)) + id1 = int(out) + id2 = int(id(interp)) + self.assertNotEqual(id1, id2) + class ListAllTests(TestBase): @@ -191,6 +238,57 @@ def test_after_destroying(self): self.assertEqual(ids, [main.id, second.id]) + def test_idempotent(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + expected = [main, first, second] + + actual = interpreters.list_all() + + self.assertEqual(actual, expected) + for interp1, interp2 in zip(actual, expected): + self.assertIs(interp1, interp2) + + +class InterpreterInitTests(TestBase): + + def test_int(self): + interpid = int(interpreters.get_current().id) + interp = interpreters.Interpreter(interpid) + self.assertEqual(int(interp.id), interpid) + + def test_interpreter_id(self): + interpid = interpreters.get_current().id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp.id, interpid) + + def test_unsupported(self): + actualid = int(interpreters.get_current().id) + for interpid in [ + str(actualid), + float(actualid), + object(), + None, + '', + ]: + with self.subTest(repr(interpid)): + with self.assertRaises(TypeError): + interpreters.Interpreter(interpid) + + def test_idempotent(self): + main = interpreters.get_main() + interp = interpreters.Interpreter(int(main.id)) + self.assertIs(interp, main) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.Interpreter(1_000_000) + + def test_bad_id(self): + with self.assertRaises(ValueError): + interpreters.Interpreter(-1) + class TestInterpreterAttrs(TestBase): @@ -202,19 +300,10 @@ def test_id_type(self): self.assertIsInstance(current.id, _interpreters.InterpreterID) self.assertIsInstance(interp.id, _interpreters.InterpreterID) - def test_main_id(self): - main = interpreters.get_main() - self.assertEqual(main.id, 0) - - def test_custom_id(self): - interp = interpreters.Interpreter(1) - self.assertEqual(interp.id, 1) - - with self.assertRaises(TypeError): - interpreters.Interpreter('1') - def test_id_readonly(self): - interp = interpreters.Interpreter(1) + actual = interpreters.create() + interpid = int(actual.id) + interp = interpreters.Interpreter(interpid) with self.assertRaises(AttributeError): interp.id = 2 @@ -236,17 +325,23 @@ def test_subinterpreter_isolated_explicit(self): @unittest.skip('not ready yet (see bpo-32604)') def test_custom_isolated_default(self): - interp = interpreters.Interpreter(1) + actual = interpreters.create() + interpid = int(actual.id) + interp = interpreters.Interpreter(interpid) self.assertFalse(interp.isolated) def test_custom_isolated_explicit(self): - interp1 = interpreters.Interpreter(1, isolated=True) - interp2 = interpreters.Interpreter(1, isolated=False) + actual = interpreters.create() + interpid = int(actual.id) + interp1 = interpreters.Interpreter(interpid, isolated=True) + interp2 = interpreters.Interpreter(interpid, isolated=False) self.assertTrue(interp1.isolated) - self.assertFalse(interp2.isolated) + self.assertTrue(interp2.isolated) def test_isolated_readonly(self): - interp = interpreters.Interpreter(1) + actual = interpreters.create() + interpid = int(actual.id) + interp = interpreters.Interpreter(interpid) with self.assertRaises(AttributeError): interp.isolated = True @@ -299,16 +394,6 @@ def test_already_destroyed(self): with self.assertRaises(RuntimeError): interp.is_running() - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.is_running() - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.is_running() - def test_with_only_background_threads(self): r_interp, w_interp = self.pipe() r_thread, w_thread = self.pipe() @@ -378,16 +463,6 @@ def test_already_destroyed(self): with self.assertRaises(RuntimeError): interp.close() - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.close() - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.close() - def test_from_current(self): main, = interpreters.list_all() interp = interpreters.create() @@ -527,16 +602,6 @@ def test_already_running(self): with self.assertRaises(RuntimeError): interp.run('print("spam")') - def test_does_not_exist(self): - interp = interpreters.Interpreter(1_000_000) - with self.assertRaises(RuntimeError): - interp.run('print("spam")') - - def test_bad_id(self): - interp = interpreters.Interpreter(-1) - with self.assertRaises(ValueError): - interp.run('print("spam")') - def test_bad_script(self): interp = interpreters.create() with self.assertRaises(TypeError): From 57a3c194bd3688548c53cc6281909e1d4326d75d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2023 14:49:32 -0700 Subject: [PATCH 02/26] Interpreters are always isolated. --- Lib/test/support/interpreters.py | 19 +++--------- Lib/test/test_interpreters.py | 53 ++++---------------------------- 2 files changed, 11 insertions(+), 61 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index f82581146d5c26..48c80f6d1eee4f 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -36,10 +36,10 @@ def __init__(self, excinfo): self.snapshot = excinfo -def create(*, isolated=True): +def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=isolated) - return Interpreter(id, isolated=isolated) + id = _interpreters.create(isolated=True) + return Interpreter(id) def list_all(): @@ -64,7 +64,7 @@ def get_main(): class Interpreter: """A single Python interpreter.""" - def __new__(cls, id, /, *, isolated=None): + def __new__(cls, id, /): # There is only one instance for any given ID. if isinstance(id, int): id = _interpreters.InterpreterID(id, force=False) @@ -76,12 +76,11 @@ def __new__(cls, id, /, *, isolated=None): except KeyError: self = super().__new__(cls) self._id = id - self._isolated = bool(isolated) _known[key] = self return self def __repr__(self): - data = dict(id=int(self._id), isolated=self._isolated) + data = dict(id=int(self._id)) kwargs = (f'{k}={v!r}' for k, v in data.items()) return f'{type(self).__name__}({", ".join(kwargs)})' @@ -92,14 +91,6 @@ def __hash__(self): def id(self): return self._id - @property - def isolated(self): - if self._isolated is None: - # XXX The low-level function has not been added yet. - # See bpo-.... - self._isolated = _interpreters.is_isolated(self._id) - return self._isolated - def is_running(self): """Return whether or not the identified interpreter is running.""" return _interpreters.is_running(self._id) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index e3158b690de868..fb0fde362bc70c 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -251,19 +251,19 @@ def test_idempotent(self): self.assertIs(interp1, interp2) -class InterpreterInitTests(TestBase): +class InterpreterObjectTests(TestBase): - def test_int(self): + def test_init_int(self): interpid = int(interpreters.get_current().id) interp = interpreters.Interpreter(interpid) self.assertEqual(int(interp.id), interpid) - def test_interpreter_id(self): + def test_init_interpreter_id(self): interpid = interpreters.get_current().id interp = interpreters.Interpreter(interpid) self.assertEqual(interp.id, interpid) - def test_unsupported(self): + def test_init_unsupported(self): actualid = int(interpreters.get_current().id) for interpid in [ str(actualid), @@ -281,17 +281,14 @@ def test_idempotent(self): interp = interpreters.Interpreter(int(main.id)) self.assertIs(interp, main) - def test_does_not_exist(self): + def test_init_does_not_exist(self): with self.assertRaises(RuntimeError): interpreters.Interpreter(1_000_000) - def test_bad_id(self): + def test_init_bad_id(self): with self.assertRaises(ValueError): interpreters.Interpreter(-1) - -class TestInterpreterAttrs(TestBase): - def test_id_type(self): main = interpreters.get_main() current = interpreters.get_current() @@ -307,44 +304,6 @@ def test_id_readonly(self): with self.assertRaises(AttributeError): interp.id = 2 - @unittest.skip('not ready yet (see bpo-32604)') - def test_main_isolated(self): - main = interpreters.get_main() - self.assertFalse(main.isolated) - - @unittest.skip('not ready yet (see bpo-32604)') - def test_subinterpreter_isolated_default(self): - interp = interpreters.create() - self.assertFalse(interp.isolated) - - def test_subinterpreter_isolated_explicit(self): - interp1 = interpreters.create(isolated=True) - interp2 = interpreters.create(isolated=False) - self.assertTrue(interp1.isolated) - self.assertFalse(interp2.isolated) - - @unittest.skip('not ready yet (see bpo-32604)') - def test_custom_isolated_default(self): - actual = interpreters.create() - interpid = int(actual.id) - interp = interpreters.Interpreter(interpid) - self.assertFalse(interp.isolated) - - def test_custom_isolated_explicit(self): - actual = interpreters.create() - interpid = int(actual.id) - interp1 = interpreters.Interpreter(interpid, isolated=True) - interp2 = interpreters.Interpreter(interpid, isolated=False) - self.assertTrue(interp1.isolated) - self.assertTrue(interp2.isolated) - - def test_isolated_readonly(self): - actual = interpreters.create() - interpid = int(actual.id) - interp = interpreters.Interpreter(interpid) - with self.assertRaises(AttributeError): - interp.isolated = True - def test_equality(self): interp1 = interpreters.create() interp2 = interpreters.create() From 747e5425a8f85e9a4fea8ace7b7bdb125f9ab461 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2023 14:55:34 -0700 Subject: [PATCH 03/26] interp.id is always int. --- Lib/test/support/interpreters.py | 2 +- Lib/test/test_interpreters.py | 30 ++++++++++++++---------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index 48c80f6d1eee4f..16597967d66cf0 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -89,7 +89,7 @@ def __hash__(self): @property def id(self): - return self._id + return int(self._id) def is_running(self): """Return whether or not the identified interpreter is running.""" diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index fb0fde362bc70c..df0c30fa199b6e 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -205,7 +205,7 @@ def test_idempotent(self): print(id(cur)) """)) id1 = int(out) - id2 = int(id(interp)) + id2 = id(interp) self.assertNotEqual(id1, id2) @@ -254,17 +254,17 @@ def test_idempotent(self): class InterpreterObjectTests(TestBase): def test_init_int(self): - interpid = int(interpreters.get_current().id) + interpid = interpreters.get_current().id interp = interpreters.Interpreter(interpid) - self.assertEqual(int(interp.id), interpid) + self.assertEqual(interp.id, interpid) def test_init_interpreter_id(self): - interpid = interpreters.get_current().id + interpid = interpreters.get_current()._id interp = interpreters.Interpreter(interpid) - self.assertEqual(interp.id, interpid) + self.assertEqual(interp._id, interpid) def test_init_unsupported(self): - actualid = int(interpreters.get_current().id) + actualid = interpreters.get_current().id for interpid in [ str(actualid), float(actualid), @@ -278,7 +278,7 @@ def test_init_unsupported(self): def test_idempotent(self): main = interpreters.get_main() - interp = interpreters.Interpreter(int(main.id)) + interp = interpreters.Interpreter(main.id) self.assertIs(interp, main) def test_init_does_not_exist(self): @@ -293,16 +293,14 @@ def test_id_type(self): main = interpreters.get_main() current = interpreters.get_current() interp = interpreters.create() - self.assertIsInstance(main.id, _interpreters.InterpreterID) - self.assertIsInstance(current.id, _interpreters.InterpreterID) - self.assertIsInstance(interp.id, _interpreters.InterpreterID) + self.assertIsInstance(main.id, int) + self.assertIsInstance(current.id, int) + self.assertIsInstance(interp.id, int) def test_id_readonly(self): - actual = interpreters.create() - interpid = int(actual.id) - interp = interpreters.Interpreter(interpid) + interp = interpreters.create() with self.assertRaises(AttributeError): - interp.id = 2 + interp.id = 1_000_000 def test_equality(self): interp1 = interpreters.create() @@ -427,7 +425,7 @@ def test_from_current(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" from test.support import interpreters - interp = interpreters.Interpreter({int(interp.id)}) + interp = interpreters.Interpreter({interp.id}) try: interp.close() except RuntimeError: @@ -444,7 +442,7 @@ def test_from_sibling(self): {main, interp1, interp2}) interp1.run(dedent(f""" from test.support import interpreters - interp2 = interpreters.Interpreter(int({interp2.id})) + interp2 = interpreters.Interpreter({interp2.id}) interp2.close() interp3 = interpreters.create() interp3.close() From 0ed9fb0684cc39465aa871b9c391ba0d550fd2a9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Dec 2023 14:05:05 -0700 Subject: [PATCH 04/26] Add InterpreterNotFoundError. --- Include/internal/pycore_crossinterp.h | 10 +++++ Lib/test/support/interpreters.py | 7 ++- Lib/test/test_interpreters.py | 7 +-- Modules/_xxsubinterpretersmodule.c | 8 ++++ Python/crossinterp.c | 62 +++++++++++++++++++++++++++ Python/pylifecycle.c | 6 +++ Python/pystate.c | 2 +- 7 files changed, 96 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index ec9dac96292f35..cb006bb5a51770 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -10,6 +10,13 @@ extern "C" { #include "pycore_pyerrors.h" +/**************/ +/* exceptions */ +/**************/ + +PyAPI_DATA(PyObject *) PyExc_InterpreterError; +PyAPI_DATA(PyObject *) PyExc_InterpreterNotFoundError; + /***************************/ /* cross-interpreter calls */ @@ -159,6 +166,9 @@ struct _xi_state { extern PyStatus _PyXI_Init(PyInterpreterState *interp); extern void _PyXI_Fini(PyInterpreterState *interp); +extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp); +extern void _PyXI_FiniTypes(PyInterpreterState *interp); + /***************************/ /* short-term data sharing */ diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index 16597967d66cf0..d956954718c06d 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -6,7 +6,10 @@ import _xxinterpchannels as _channels # aliases: -from _xxsubinterpreters import is_shareable +from _xxsubinterpreters import ( + InterpreterError, InterpreterNotFoundError, + is_shareable, +) from _xxinterpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, @@ -15,7 +18,7 @@ __all__ = [ 'Interpreter', 'get_current', 'get_main', 'create', 'list_all', - 'RunFailedError', + 'InterpreterError', 'InterpreterNotFoundError', 'RunFailedError', 'SendChannel', 'RecvChannel', 'create_channel', 'list_all_channels', 'is_shareable', 'ChannelError', 'ChannelNotFoundError', diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index df0c30fa199b6e..02c4722ab9bdcb 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -15,6 +15,7 @@ _interpreters = import_helper.import_module('_xxsubinterpreters') _channels = import_helper.import_module('_xxinterpchannels') from test.support import interpreters +from test.support.interpreters import InterpreterNotFoundError def _captured_script(script): @@ -282,7 +283,7 @@ def test_idempotent(self): self.assertIs(interp, main) def test_init_does_not_exist(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.Interpreter(1_000_000) def test_init_bad_id(self): @@ -348,7 +349,7 @@ def test_from_subinterpreter(self): def test_already_destroyed(self): interp = interpreters.create() interp.close() - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interp.is_running() def test_with_only_background_threads(self): @@ -417,7 +418,7 @@ def f(): def test_already_destroyed(self): interp = interpreters.create() interp.close() - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interp.close() def test_from_current(self): diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 02c2abed27ddfa..f803f77bc81fc6 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -725,6 +725,14 @@ module_exec(PyObject *mod) goto error; } + // exceptions + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { + goto error; + } + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { + goto error; + } + return 0; error: diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 21b96ef05ed799..f27c7a9e8e32da 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -12,6 +12,53 @@ #include "pycore_weakref.h" // _PyWeakref_GET_REF() +/**************/ +/* exceptions */ +/**************/ + +/* InterpreterError extends Exception */ + +static PyTypeObject _PyExc_InterpreterError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "InterpreterError", + .tp_doc = PyDoc_STR("An interpreter was not found."), + //.tp_base = (PyTypeObject *)PyExc_BaseException, +}; +PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; + +/* InterpreterNotFoundError extends InterpreterError */ + +static PyTypeObject _PyExc_InterpreterNotFoundError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "InterpreterNotFoundError", + .tp_doc = PyDoc_STR("An interpreter was not found."), + .tp_base = &_PyExc_InterpreterError, +}; +PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; + +/* lifecycle */ + +static int +init_exceptions(PyInterpreterState *interp) +{ + _PyExc_InterpreterError.tp_base = (PyTypeObject *)PyExc_BaseException; + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { + return -1; + } + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { + return -1; + } + return 0; +} + +static void +fini_exceptions(PyInterpreterState *interp) +{ + _PyStaticType_Dealloc(interp, &_PyExc_InterpreterNotFoundError); + _PyStaticType_Dealloc(interp, &_PyExc_InterpreterError); +} + + /***************************/ /* cross-interpreter calls */ /***************************/ @@ -2118,3 +2165,18 @@ _PyXI_Fini(PyInterpreterState *interp) _xidregistry_fini(_get_global_xidregistry(interp->runtime)); } } + +PyStatus +_PyXI_InitTypes(PyInterpreterState *interp) +{ + if (init_exceptions(interp) < 0) { + return _PyStatus_ERR("failed to initialize an exception type"); + } + return _PyStatus_OK(); +} + +void +_PyXI_FiniTypes(PyInterpreterState *interp) +{ + fini_exceptions(interp); +} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ac8d5208322882..acb69f6d8982d0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -739,6 +739,11 @@ pycore_init_types(PyInterpreterState *interp) return status; } + status = _PyXI_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } @@ -1742,6 +1747,7 @@ finalize_interp_types(PyInterpreterState *interp) { _PyUnicode_FiniTypes(interp); _PySys_FiniTypes(interp); + _PyXI_FiniTypes(interp); _PyExc_Fini(interp); _PyAsyncGen_Fini(interp); _PyContext_Fini(interp); diff --git a/Python/pystate.c b/Python/pystate.c index 6196b15da0117a..9f37d7b2cd97d2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1309,7 +1309,7 @@ _PyInterpreterState_LookUpID(int64_t requested_id) HEAD_UNLOCK(runtime); } if (interp == NULL && !PyErr_Occurred()) { - PyErr_Format(PyExc_RuntimeError, + PyErr_Format(PyExc_InterpreterNotFoundError, "unrecognized interpreter ID %lld", requested_id); } return interp; From 98f9cf388640856fcf97af3d5238b893d931a49b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Dec 2023 14:47:17 -0700 Subject: [PATCH 05/26] Stop using InterpreterID in _interpreters. --- Include/internal/pycore_interp.h | 6 +- Lib/test/support/interpreters.py | 38 +++++-- Lib/test/test__xxinterpchannels.py | 3 +- Lib/test/test__xxsubinterpreters.py | 82 ++------------ Lib/test/test_capi/test_misc.py | 131 ++++++++++++++++++++++ Lib/test/test_interpreters.py | 6 +- Modules/_testcapimodule.c | 34 ++++++ Modules/_testinternalcapi.c | 12 ++ Modules/_xxsubinterpretersmodule.c | 168 +++++++++++++++++++++++++--- 9 files changed, 378 insertions(+), 102 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 498db8becf114c..b27ee0a8b882e1 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -249,9 +249,9 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst // Export for the _xxinterpchannels module. PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); -extern int _PyInterpreterState_IDInitref(PyInterpreterState *); -extern int _PyInterpreterState_IDIncref(PyInterpreterState *); -extern void _PyInterpreterState_IDDecref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index d956954718c06d..cf72aa5f44a4a1 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -69,30 +69,48 @@ class Interpreter: def __new__(cls, id, /): # There is only one instance for any given ID. - if isinstance(id, int): - id = _interpreters.InterpreterID(id, force=False) - elif not isinstance(id, _interpreters.InterpreterID): + if not isinstance(id, int): raise TypeError(f'id must be an int, got {id!r}') - key = int(id) + id = int(id) try: - self = _known[key] + self = _known[id] + assert hasattr(self, '_ownsref') except KeyError: - self = super().__new__(cls) - self._id = id - _known[key] = self + # This may raise InterpreterNotFoundError: + _interpreters._incref(id) + try: + self = super().__new__(cls) + self._id = id + self._ownsref = True + except BaseException: + _interpreters._deccref(id) + raise + _known[id] = self return self def __repr__(self): - data = dict(id=int(self._id)) + data = dict(id=self._id) kwargs = (f'{k}={v!r}' for k, v in data.items()) return f'{type(self).__name__}({", ".join(kwargs)})' def __hash__(self): return hash(self._id) + def __del__(self): + self._decref() + + def _decref(self): + if not self._ownsref: + return + self._ownsref = False + try: + _interpreters._decref(self.id) + except InterpreterNotFoundError: + pass + @property def id(self): - return int(self._id) + return self._id def is_running(self): """Return whether or not the identified interpreter is running.""" diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 2b75e2f1916c82..13c8a10296e502 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -79,8 +79,7 @@ def __new__(cls, name=None, id=None): name = 'interp' elif name == 'main': raise ValueError('name mismatch (unexpected "main")') - if not isinstance(id, interpreters.InterpreterID): - id = interpreters.InterpreterID(id) + assert isinstance(id, int), repr(id) elif not name or name == 'main': name = 'main' id = main diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 64a9db95e5eaf5..260ab64b07cb2d 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -15,6 +15,7 @@ interpreters = import_helper.import_module('_xxsubinterpreters') +from _xxsubinterpreters import InterpreterNotFoundError ################################## @@ -266,7 +267,7 @@ def test_main(self): main = interpreters.get_main() cur = interpreters.get_current() self.assertEqual(cur, main) - self.assertIsInstance(cur, interpreters.InterpreterID) + self.assertIsInstance(cur, int) def test_subinterpreter(self): main = interpreters.get_main() @@ -275,7 +276,7 @@ def test_subinterpreter(self): import _xxsubinterpreters as _interpreters cur = _interpreters.get_current() print(cur) - assert isinstance(cur, _interpreters.InterpreterID) + assert isinstance(cur, int) """)) cur = int(out.strip()) _, expected = interpreters.list_all() @@ -289,7 +290,7 @@ def test_from_main(self): [expected] = interpreters.list_all() main = interpreters.get_main() self.assertEqual(main, expected) - self.assertIsInstance(main, interpreters.InterpreterID) + self.assertIsInstance(main, int) def test_from_subinterpreter(self): [expected] = interpreters.list_all() @@ -298,7 +299,7 @@ def test_from_subinterpreter(self): import _xxsubinterpreters as _interpreters main = _interpreters.get_main() print(main) - assert isinstance(main, _interpreters.InterpreterID) + assert isinstance(main, int) """)) main = int(out.strip()) self.assertEqual(main, expected) @@ -333,11 +334,11 @@ def test_from_subinterpreter(self): def test_already_destroyed(self): interp = interpreters.create() interpreters.destroy(interp) - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.is_running(interp) def test_does_not_exist(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.is_running(1_000_000) def test_bad_id(self): @@ -345,70 +346,11 @@ def test_bad_id(self): interpreters.is_running(-1) -class InterpreterIDTests(TestBase): - - def test_with_int(self): - id = interpreters.InterpreterID(10, force=True) - - self.assertEqual(int(id), 10) - - def test_coerce_id(self): - class Int(str): - def __index__(self): - return 10 - - id = interpreters.InterpreterID(Int(), force=True) - self.assertEqual(int(id), 10) - - def test_bad_id(self): - self.assertRaises(TypeError, interpreters.InterpreterID, object()) - self.assertRaises(TypeError, interpreters.InterpreterID, 10.0) - self.assertRaises(TypeError, interpreters.InterpreterID, '10') - self.assertRaises(TypeError, interpreters.InterpreterID, b'10') - self.assertRaises(ValueError, interpreters.InterpreterID, -1) - self.assertRaises(OverflowError, interpreters.InterpreterID, 2**64) - - def test_does_not_exist(self): - id = interpreters.create() - with self.assertRaises(RuntimeError): - interpreters.InterpreterID(int(id) + 1) # unforced - - def test_str(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = interpreters.create() - id2 = interpreters.InterpreterID(int(id1)) - id3 = interpreters.create() - - self.assertTrue(id1 == id1) - self.assertTrue(id1 == id2) - self.assertTrue(id1 == int(id1)) - self.assertTrue(int(id1) == id1) - self.assertTrue(id1 == float(int(id1))) - self.assertTrue(float(int(id1)) == id1) - self.assertFalse(id1 == float(int(id1)) + 0.1) - self.assertFalse(id1 == str(int(id1))) - self.assertFalse(id1 == 2**1000) - self.assertFalse(id1 == float('inf')) - self.assertFalse(id1 == 'spam') - self.assertFalse(id1 == id3) - - self.assertFalse(id1 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id1 != id3) - - class CreateTests(TestBase): def test_in_main(self): id = interpreters.create() - self.assertIsInstance(id, interpreters.InterpreterID) + self.assertIsInstance(id, int) self.assertIn(id, interpreters.list_all()) @@ -444,7 +386,7 @@ def test_in_subinterpreter(self): import _xxsubinterpreters as _interpreters id = _interpreters.create() print(id) - assert isinstance(id, _interpreters.InterpreterID) + assert isinstance(id, int) """)) id2 = int(out.strip()) @@ -536,11 +478,11 @@ def f(): def test_already_destroyed(self): id = interpreters.create() interpreters.destroy(id) - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.destroy(id) def test_does_not_exist(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.destroy(1_000_000) def test_bad_id(self): @@ -741,7 +683,7 @@ def test_does_not_exist(self): id = 0 while id in interpreters.list_all(): id += 1 - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterNotFoundError): interpreters.run_string(id, 'print("spam")') def test_error_id(self): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 6cbf5d22203804..dec68977a6ba45 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2020,6 +2020,137 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +@requires_subinterpreters +class InterpreterIDTests(unittest.TestCase): + + InterpreterID = _testcapi.get_interpreterid_type() + + def new_interpreter(self): + def ensure_destroyed(interpid): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + id = _interpreters.create() + self.addCleanup(lambda: ensure_destroyed(id)) + return id + + def test_with_int(self): + id = self.InterpreterID(10, force=True) + + self.assertEqual(int(id), 10) + + def test_coerce_id(self): + class Int(str): + def __index__(self): + return 10 + + id = self.InterpreterID(Int(), force=True) + self.assertEqual(int(id), 10) + + def test_bad_id(self): + for badid in [ + object(), + 10.0, + '10', + b'10', + ]: + with self.subTest(badid): + with self.assertRaises(TypeError): + self.InterpreterID(badid) + + badid = -1 + with self.subTest(badid): + with self.assertRaises(ValueError): + self.InterpreterID(badid) + + badid = 2**64 + with self.subTest(badid): + with self.assertRaises(OverflowError): + self.InterpreterID(badid) + + def test_exists(self): + id = self.new_interpreter() + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(int(id) + 1) # unforced + + def test_does_not_exist(self): + id = self.new_interpreter() + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(int(id) + 1) # unforced + + def test_does_not_exist(self): + id = _interpreters.create() + _interpreters.destroy(id) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + self.InterpreterID(id) # unforced + + def test_str(self): + id = self.InterpreterID(10, force=True) + self.assertEqual(str(id), '10') + + def test_repr(self): + id = self.InterpreterID(10, force=True) + self.assertEqual(repr(id), 'InterpreterID(10)') + + def test_equality(self): + id1 = self.new_interpreter() + id2 = self.InterpreterID(id1) + id3 = self.InterpreterID( + self.new_interpreter()) + + self.assertTrue(id2 == id2) # identity + self.assertTrue(id2 == id1) # int-equivalent + self.assertTrue(id1 == id2) # reversed + self.assertTrue(id2 == int(id2)) + self.assertTrue(id2 == float(int(id2))) + self.assertTrue(float(int(id2)) == id2) + self.assertFalse(id2 == float(int(id2)) + 0.1) + self.assertFalse(id2 == str(int(id2))) + self.assertFalse(id2 == 2**1000) + self.assertFalse(id2 == float('inf')) + self.assertFalse(id2 == 'spam') + self.assertFalse(id2 == id3) + + self.assertFalse(id2 != id2) + self.assertFalse(id2 != id1) + self.assertFalse(id1 != id2) + self.assertTrue(id2 != id3) + + def test_linked_lifecycle(self): + id1 = _interpreters.create() + _testcapi.unlink_interpreter_refcount(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + id2 = self.InterpreterID(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 1) + + # The interpreter isn't linked to ID objects, so it isn't destroyed. + del id2 + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + _testcapi.link_interpreter_refcount(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 0) + + id3 = self.InterpreterID(id1) + self.assertEqual( + _testinternalcapi.get_interpreter_refcount(id1), + 1) + + # The interpreter is linked now so is destroyed. + del id3 + with self.assertRaises(_interpreters.InterpreterNotFoundError): + _testinternalcapi.get_interpreter_refcount(id1) + + class BuiltinStaticTypesTests(unittest.TestCase): TYPES = [ diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 02c4722ab9bdcb..4e37da92b31bca 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -778,10 +778,12 @@ def test_sys_path_0(self): class FinalizationTests(TestBase): def test_gh_109793(self): + # Make sure finalization finishes and the correct error code + # is reported, even when subinterpreters get cleaned up at the end. import subprocess argv = [sys.executable, '-c', '''if True: - import _xxsubinterpreters as _interpreters - interpid = _interpreters.create() + from test.support import interpreters + interp = interpreters.create() raise Exception '''] proc = subprocess.run(argv, capture_output=True, text=True) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9fdd67093338e4..b3ddfae58e6fc0 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -13,6 +13,7 @@ #include "_testcapi/parts.h" #include "frameobject.h" // PyFrame_New() +#include "interpreteridobject.h" // PyInterpreterID_Type #include "marshal.h" // PyMarshal_WriteLongToFile() #include // FLT_MAX @@ -1451,6 +1452,36 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } +static PyObject * +get_interpreterid_type(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return Py_NewRef(&PyInterpreterID_Type); +} + +static PyObject * +link_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 1); + Py_RETURN_NONE; +} + +static PyObject * +unlink_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 0); + Py_RETURN_NONE; +} + static PyMethodDef ml; static PyObject * @@ -3237,6 +3268,9 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, + {"get_interpreterid_type", get_interpreterid_type, METH_NOARGS}, + {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, + {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 4607a3faf17f74..25088af5786806 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1475,6 +1475,17 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } +static PyObject * +get_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + if (interp == NULL) { + return NULL; + } + return PyLong_FromLongLong(interp->id_refcount); +} + + static void _xid_capsule_destructor(PyObject *capsule) { @@ -1682,6 +1693,7 @@ static PyMethodDef module_functions[] = { {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, + {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index f803f77bc81fc6..5954d47e2555ec 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -6,11 +6,13 @@ #endif #include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_crossinterp.h" // struct _xid -#include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() -#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pyerrors.h" // _Py_excinfo #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "interpreteridobject.h" @@ -28,6 +30,97 @@ _get_current_interp(void) return PyInterpreterState_Get(); } +static int64_t +pylong_to_interpid(PyObject *idobj) +{ + assert(PyLong_CheckExact(idobj)); + + if (_PyLong_IsNegative((PyLongObject *)idobj)) { + PyErr_Format(PyExc_ValueError, + "interpreter ID must be a non-negative int, got %R", + idobj); + return -1; + } + + int overflow; + long long id = PyLong_AsLongLongAndOverflow(idobj, &overflow); + if (id == -1) { + if (!overflow) { + assert(PyErr_Occurred()); + return -1; + } + assert(!PyErr_Occurred()); + // For now, we don't worry about if LLONG_MAX < INT64_MAX. + goto bad_id; + } +#if LONG_LONG_MAX > INT64_MAX + if (id > INT64_MAX) { + goto bad_id; + } +#endif + return (int64_t)id; + +bad_id: + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %O", idobj); + return -1; +} + +static int64_t +convert_interpid_obj(PyObject *arg) +{ + int64_t id = -1; + if (_PyIndex_Check(arg)) { + PyObject *idobj = PyNumber_Long(arg); + if (idobj == NULL) { + return -1; + } + id = pylong_to_interpid(idobj); + Py_DECREF(idobj); + if (id < 0) { + return -1; + } + } + else { + PyErr_Format(PyExc_TypeError, + "interpreter ID must be an int, got %.100s", + Py_TYPE(arg)->tp_name); + return -1; + } + return id; +} + +static PyInterpreterState * +look_up_interp(PyObject *arg) +{ + int64_t id = convert_interpid_obj(arg); + if (id < 0) { + return NULL; + } + return _PyInterpreterState_LookUpID(id); +} + + +static PyObject * +interpid_to_pylong(int64_t id) +{ + assert(id < LONG_LONG_MAX); + return PyLong_FromLongLong(id); +} + +static PyObject * +get_interpid_obj(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IDInitref(interp) != 0) { + return NULL; + }; + int64_t id = PyInterpreterState_GetID(interp); + if (id < 0) { + return NULL; + } + return interpid_to_pylong(id); +} + /* module state *************************************************************/ @@ -254,7 +347,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) assert(tstate != NULL); PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = PyInterpreterState_GetIDObject(interp); + PyObject *idobj = get_interpid_obj(interp); if (idobj == NULL) { // XXX Possible GILState issues? save_tstate = PyThreadState_Swap(tstate); @@ -273,7 +366,9 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) PyDoc_STRVAR(create_doc, "create() -> ID\n\ \n\ -Create a new interpreter and return a unique generated ID."); +Create a new interpreter and return a unique generated ID.\n\ +\n\ +The caller is responsible for destroying the interpreter before exiting."); static PyObject * @@ -288,7 +383,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; } @@ -345,7 +440,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) interp = PyInterpreterState_Head(); while (interp != NULL) { - id = PyInterpreterState_GetIDObject(interp); + id = get_interpid_obj(interp); if (id == NULL) { Py_DECREF(ids); return NULL; @@ -377,7 +472,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } - return PyInterpreterState_GetIDObject(interp); + return get_interpid_obj(interp); } PyDoc_STRVAR(get_current_doc, @@ -391,7 +486,7 @@ interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) { // Currently, 0 is always the main interpreter. int64_t id = 0; - return PyInterpreterID_New(id); + return PyLong_FromLongLong(id); } PyDoc_STRVAR(get_main_doc, @@ -481,7 +576,7 @@ _interp_exec(PyObject *self, PyObject **p_excinfo) { // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id_arg); + PyInterpreterState *interp = look_up_interp(id_arg); if (interp == NULL) { return -1; } @@ -667,7 +762,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; } @@ -683,6 +778,49 @@ PyDoc_STRVAR(is_running_doc, Return whether or not the identified interpreter is running."); +static PyObject * +interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *id; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:_incref", kwlist, &id)) { + return NULL; + } + + PyInterpreterState *interp = look_up_interp(id); + if (interp == NULL) { + return NULL; + } + if (_PyInterpreterState_IDInitref(interp) < 0) { + return NULL; + } + _PyInterpreterState_IDIncref(interp); + + Py_RETURN_NONE; +} + + +static PyObject * +interp_decref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *id; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:_incref", kwlist, &id)) { + return NULL; + } + + PyInterpreterState *interp = look_up_interp(id); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDDecref(interp); + + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, @@ -707,6 +845,11 @@ static PyMethodDef module_functions[] = { {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + {"_incref", _PyCFunction_CAST(interp_incref), + METH_VARARGS | METH_KEYWORDS, NULL}, + {"_decref", _PyCFunction_CAST(interp_decref), + METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL} /* sentinel */ }; @@ -720,11 +863,6 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { - // PyInterpreterID - if (PyModule_AddType(mod, &PyInterpreterID_Type) < 0) { - goto error; - } - // exceptions if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { goto error; From 0277878313085738c7f14df1667f4532502b6b56 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2023 14:57:14 -0700 Subject: [PATCH 06/26] Fix Interpreter.__repr__(). --- Lib/test/support/interpreters.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index cf72aa5f44a4a1..ba62246d123118 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -89,9 +89,7 @@ def __new__(cls, id, /): return self def __repr__(self): - data = dict(id=self._id) - kwargs = (f'{k}={v!r}' for k, v in data.items()) - return f'{type(self).__name__}({", ".join(kwargs)})' + return f'{type(self).__name__}({self.id})' def __hash__(self): return hash(self._id) From 59c8227c72e67dde383280010e05b67b65ccf6b2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2023 15:03:14 -0700 Subject: [PATCH 07/26] .run() -> .exec_sync() --- Lib/test/support/interpreters.py | 5 ++-- Lib/test/test_interpreters.py | 43 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index ba62246d123118..9a4680291ea103 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -122,8 +122,7 @@ def close(self): """ return _interpreters.destroy(self._id) - # XXX Rename "run" to "exec"? - def run(self, src_str, /, channels=None): + def exec_sync(self, code, /, channels=None): """Run the given source code in the interpreter. This is essentially the same as calling the builtin "exec" @@ -141,7 +140,7 @@ def run(self, src_str, /, channels=None): that time, the previous interpreter is allowed to run in other threads. """ - excinfo = _interpreters.exec(self._id, src_str, channels) + excinfo = _interpreters.exec(self._id, code, channels) if excinfo is not None: raise RunFailedError(excinfo) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 4e37da92b31bca..49f8b9a6f80c29 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -43,7 +43,7 @@ def clean_up_interpreters(): def _run_output(interp, request, channels=None): script, rpipe = _captured_script(request) with rpipe: - interp.run(script, channels=channels) + interp.exec_sync(script, channels=channels) return rpipe.read() @@ -51,7 +51,7 @@ def _run_output(interp, request, channels=None): def _running(interp): r, w = os.pipe() def run(): - interp.run(dedent(f""" + interp.exec_sync(dedent(f""" # wait for "signal" with open({r}) as rpipe: rpipe.read() @@ -328,7 +328,7 @@ def test_subinterpreter(self): def test_finished(self): r, w = self.pipe() interp = interpreters.create() - interp.run(f"""if True: + interp.exec_sync(f"""if True: import os os.write({w}, b'x') """) @@ -360,7 +360,7 @@ def test_with_only_background_threads(self): FINISHED = b'F' interp = interpreters.create() - interp.run(f"""if True: + interp.exec_sync(f"""if True: import os import threading @@ -374,7 +374,7 @@ def task(): self.assertFalse(interp.is_running()) os.write(w_thread, DONE) - interp.run('t.join()') + interp.exec_sync('t.join()') self.assertEqual(os.read(r_interp, 1), FINISHED) @@ -441,7 +441,7 @@ def test_from_sibling(self): interp2 = interpreters.create() self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) - interp1.run(dedent(f""" + interp1.exec_sync(dedent(f""" from test.support import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() @@ -475,7 +475,7 @@ def test_subthreads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.run(f"""if True: + interp.exec_sync(f"""if True: import os import threading import time @@ -506,7 +506,7 @@ def test_success(self): interp = interpreters.create() script, file = _captured_script('print("it worked!", end="")') with file: - interp.run(script) + interp.exec_sync(script) out = file.read() self.assertEqual(out, 'it worked!') @@ -514,14 +514,14 @@ def test_success(self): def test_failure(self): interp = interpreters.create() with self.assertRaises(interpreters.RunFailedError): - interp.run('raise Exception') + interp.exec_sync('raise Exception') def test_in_thread(self): interp = interpreters.create() script, file = _captured_script('print("it worked!", end="")') with file: def f(): - interp.run(script) + interp.exec_sync(script) t = threading.Thread(target=f) t.start() @@ -547,7 +547,7 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interp.run(script) + interp.exec_sync(script) file.seek(0) content = file.read() @@ -558,17 +558,17 @@ def test_already_running(self): interp = interpreters.create() with _running(interp): with self.assertRaises(RuntimeError): - interp.run('print("spam")') + interp.exec_sync('print("spam")') def test_bad_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.run(10) + interp.exec_sync(10) def test_bytes_for_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.run(b'print("spam")') + interp.exec_sync(b'print("spam")') def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -579,7 +579,7 @@ def test_with_background_threads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.run(f"""if True: + interp.exec_sync(f"""if True: import os import threading @@ -591,17 +591,18 @@ def task(): t.start() os.write({w_interp}, {RAN!r}) """) - interp.run(f"""if True: + interp.exec_sync(f"""if True: os.write({w_interp}, {RAN!r}) """) os.write(w_thread, DONE) - interp.run('t.join()') + interp.exec_sync('t.join()') self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), FINISHED) - # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. + # test_xxsubinterpreters covers the remaining + # Interpreter.exec_sync() behavior. class StressTests(TestBase): @@ -737,7 +738,7 @@ def test_sys_path_0(self): orig = sys.path[0] interp = interpreters.create() - interp.run(f"""if True: + interp.exec_sync(f"""if True: import json import sys print(json.dumps({{ @@ -958,7 +959,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() - interp.run(dedent(""" + interp.exec_sync(dedent(""" from test.support import interpreters r, s = interpreters.create_channel() orig = b'spam' @@ -1031,7 +1032,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() - interp.run(dedent(""" + interp.exec_sync(dedent(""" from test.support import interpreters r, s = interpreters.create_channel() orig = b'spam' From fa132a290b85fab376f94351d00fcfab0afb619d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2023 16:54:50 -0700 Subject: [PATCH 08/26] RunFailedError -> ExecFailure --- Lib/test/support/interpreters.py | 8 ++++---- Lib/test/test_interpreters.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index 9a4680291ea103..d2a78ccce57b74 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -18,7 +18,7 @@ __all__ = [ 'Interpreter', 'get_current', 'get_main', 'create', 'list_all', - 'InterpreterError', 'InterpreterNotFoundError', 'RunFailedError', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', 'SendChannel', 'RecvChannel', 'create_channel', 'list_all_channels', 'is_shareable', 'ChannelError', 'ChannelNotFoundError', @@ -26,7 +26,7 @@ ] -class RunFailedError(RuntimeError): +class ExecFailure(RuntimeError): def __init__(self, excinfo): msg = excinfo.formatted @@ -131,7 +131,7 @@ def exec_sync(self, code, /, channels=None): There is no return value. - If the code raises an unhandled exception then a RunFailedError + If the code raises an unhandled exception then an ExecFailure is raised, which summarizes the unhandled exception. The actual exception is discarded because objects cannot be shared between interpreters. @@ -142,7 +142,7 @@ def exec_sync(self, code, /, channels=None): """ excinfo = _interpreters.exec(self._id, code, channels) if excinfo is not None: - raise RunFailedError(excinfo) + raise ExecFailure(excinfo) def create_channel(): diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 49f8b9a6f80c29..60db7373db6dcf 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -500,7 +500,7 @@ def task(): self.assertEqual(os.read(r_interp, 1), FINISHED) -class TestInterpreterRun(TestBase): +class TestInterpreterExecSync(TestBase): def test_success(self): interp = interpreters.create() @@ -513,7 +513,7 @@ def test_success(self): def test_failure(self): interp = interpreters.create() - with self.assertRaises(interpreters.RunFailedError): + with self.assertRaises(interpreters.ExecFailure): interp.exec_sync('raise Exception') def test_in_thread(self): From 644efa59ae80df07339a006d849aa36c77a4a94b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2023 16:58:14 -0700 Subject: [PATCH 09/26] Add Interpreter.run(). --- Lib/test/support/interpreters.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index d2a78ccce57b74..1d642b3ed18de7 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -144,6 +144,13 @@ def exec_sync(self, code, /, channels=None): if excinfo is not None: raise ExecFailure(excinfo) + def run(self, code, /, channels=None): + def task(): + self.exec_sync(code, channels=channels) + t = threading.Thread(target=task) + t.start() + return t + def create_channel(): """Return (recv, send) for a new cross-interpreter channel. From abf2aa8ce2d9aeb5131803febb2f8ffe1af99041 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Dec 2023 15:39:25 -0700 Subject: [PATCH 10/26] Make the interpreters module a package. --- Lib/test/support/interpreters/__init__.py | 145 ++ .../channel.py} | 145 +- Lib/test/test_interpreters.py | 1162 ----------------- Lib/test/test_interpreters/__init__.py | 5 + Lib/test/test_interpreters/__main__.py | 4 + Lib/test/test_interpreters/test_api.py | 596 +++++++++ Lib/test/test_interpreters/test_channel.py | 328 +++++ Lib/test/test_interpreters/test_lifecycle.py | 189 +++ Lib/test/test_interpreters/test_stress.py | 38 + Lib/test/test_interpreters/utils.py | 73 ++ 10 files changed, 1383 insertions(+), 1302 deletions(-) create mode 100644 Lib/test/support/interpreters/__init__.py rename Lib/test/support/{interpreters.py => interpreters/channel.py} (54%) delete mode 100644 Lib/test/test_interpreters.py create mode 100644 Lib/test/test_interpreters/__init__.py create mode 100644 Lib/test/test_interpreters/__main__.py create mode 100644 Lib/test/test_interpreters/test_api.py create mode 100644 Lib/test/test_interpreters/test_channel.py create mode 100644 Lib/test/test_interpreters/test_lifecycle.py create mode 100644 Lib/test/test_interpreters/test_stress.py create mode 100644 Lib/test/test_interpreters/utils.py diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py new file mode 100644 index 00000000000000..86b625640a439f --- /dev/null +++ b/Lib/test/support/interpreters/__init__.py @@ -0,0 +1,145 @@ +"""Subinterpreters High Level Module.""" + +import weakref +import _xxsubinterpreters as _interpreters + +# aliases: +from _xxsubinterpreters import ( + InterpreterError, InterpreterNotFoundError, + is_shareable, +) +from .channel import create as create_channel + + +__all__ = [ + 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', + 'Interpreter', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', + 'create_channel', +] + + +class ExecFailure(RuntimeError): + + def __init__(self, excinfo): + msg = excinfo.formatted + if not msg: + if excinfo.type and snapshot.msg: + msg = f'{snapshot.type.__name__}: {snapshot.msg}' + else: + msg = snapshot.type.__name__ or snapshot.msg + super().__init__(msg) + self.snapshot = excinfo + + +def create(): + """Return a new (idle) Python interpreter.""" + id = _interpreters.create(isolated=True) + return Interpreter(id) + + +def list_all(): + """Return all existing interpreters.""" + return [Interpreter(id) for id in _interpreters.list_all()] + + +def get_current(): + """Return the currently running interpreter.""" + id = _interpreters.get_current() + return Interpreter(id) + + +def get_main(): + """Return the main interpreter.""" + id = _interpreters.get_main() + return Interpreter(id) + + +_known = weakref.WeakValueDictionary() + +class Interpreter: + """A single Python interpreter.""" + + def __new__(cls, id, /): + # There is only one instance for any given ID. + if not isinstance(id, int): + raise TypeError(f'id must be an int, got {id!r}') + id = int(id) + try: + self = _known[id] + assert hasattr(self, '_ownsref') + except KeyError: + # This may raise InterpreterNotFoundError: + _interpreters._incref(id) + try: + self = super().__new__(cls) + self._id = id + self._ownsref = True + except BaseException: + _interpreters._deccref(id) + raise + _known[id] = self + return self + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + def __del__(self): + self._decref() + + def _decref(self): + if not self._ownsref: + return + self._ownsref = False + try: + _interpreters._decref(self.id) + except InterpreterNotFoundError: + pass + + @property + def id(self): + return self._id + + def is_running(self): + """Return whether or not the identified interpreter is running.""" + return _interpreters.is_running(self._id) + + def close(self): + """Finalize and destroy the interpreter. + + Attempting to destroy the current interpreter results + in a RuntimeError. + """ + return _interpreters.destroy(self._id) + + def exec_sync(self, code, /, channels=None): + """Run the given source code in the interpreter. + + This is essentially the same as calling the builtin "exec" + with this interpreter, using the __dict__ of its __main__ + module as both globals and locals. + + There is no return value. + + If the code raises an unhandled exception then an ExecFailure + is raised, which summarizes the unhandled exception. The actual + exception is discarded because objects cannot be shared between + interpreters. + + This blocks the current Python thread until done. During + that time, the previous interpreter is allowed to run + in other threads. + """ + excinfo = _interpreters.exec(self._id, code, channels) + if excinfo is not None: + raise ExecFailure(excinfo) + + def run(self, code, /, channels=None): + def task(): + self.exec_sync(code, channels=channels) + t = threading.Thread(target=task) + t.start() + return t diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters/channel.py similarity index 54% rename from Lib/test/support/interpreters.py rename to Lib/test/support/interpreters/channel.py index 1d642b3ed18de7..75a5a60f54f926 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters/channel.py @@ -1,15 +1,9 @@ -"""Subinterpreters High Level Module.""" +"""Cross-interpreter Channels High Level Module.""" import time -import weakref -import _xxsubinterpreters as _interpreters import _xxinterpchannels as _channels # aliases: -from _xxsubinterpreters import ( - InterpreterError, InterpreterNotFoundError, - is_shareable, -) from _xxinterpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, @@ -17,142 +11,13 @@ __all__ = [ - 'Interpreter', 'get_current', 'get_main', 'create', 'list_all', - 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', + 'create', 'list_all', 'SendChannel', 'RecvChannel', - 'create_channel', 'list_all_channels', 'is_shareable', - 'ChannelError', 'ChannelNotFoundError', - 'ChannelEmptyError', - ] - - -class ExecFailure(RuntimeError): - - def __init__(self, excinfo): - msg = excinfo.formatted - if not msg: - if excinfo.type and snapshot.msg: - msg = f'{snapshot.type.__name__}: {snapshot.msg}' - else: - msg = snapshot.type.__name__ or snapshot.msg - super().__init__(msg) - self.snapshot = excinfo + 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', +] def create(): - """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=True) - return Interpreter(id) - - -def list_all(): - """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] - - -def get_current(): - """Return the currently running interpreter.""" - id = _interpreters.get_current() - return Interpreter(id) - - -def get_main(): - """Return the main interpreter.""" - id = _interpreters.get_main() - return Interpreter(id) - - -_known = weakref.WeakValueDictionary() - -class Interpreter: - """A single Python interpreter.""" - - def __new__(cls, id, /): - # There is only one instance for any given ID. - if not isinstance(id, int): - raise TypeError(f'id must be an int, got {id!r}') - id = int(id) - try: - self = _known[id] - assert hasattr(self, '_ownsref') - except KeyError: - # This may raise InterpreterNotFoundError: - _interpreters._incref(id) - try: - self = super().__new__(cls) - self._id = id - self._ownsref = True - except BaseException: - _interpreters._deccref(id) - raise - _known[id] = self - return self - - def __repr__(self): - return f'{type(self).__name__}({self.id})' - - def __hash__(self): - return hash(self._id) - - def __del__(self): - self._decref() - - def _decref(self): - if not self._ownsref: - return - self._ownsref = False - try: - _interpreters._decref(self.id) - except InterpreterNotFoundError: - pass - - @property - def id(self): - return self._id - - def is_running(self): - """Return whether or not the identified interpreter is running.""" - return _interpreters.is_running(self._id) - - def close(self): - """Finalize and destroy the interpreter. - - Attempting to destroy the current interpreter results - in a RuntimeError. - """ - return _interpreters.destroy(self._id) - - def exec_sync(self, code, /, channels=None): - """Run the given source code in the interpreter. - - This is essentially the same as calling the builtin "exec" - with this interpreter, using the __dict__ of its __main__ - module as both globals and locals. - - There is no return value. - - If the code raises an unhandled exception then an ExecFailure - is raised, which summarizes the unhandled exception. The actual - exception is discarded because objects cannot be shared between - interpreters. - - This blocks the current Python thread until done. During - that time, the previous interpreter is allowed to run - in other threads. - """ - excinfo = _interpreters.exec(self._id, code, channels) - if excinfo is not None: - raise ExecFailure(excinfo) - - def run(self, code, /, channels=None): - def task(): - self.exec_sync(code, channels=channels) - t = threading.Thread(target=task) - t.start() - return t - - -def create_channel(): """Return (recv, send) for a new cross-interpreter channel. The channel may be used to pass data safely between interpreters. @@ -162,7 +27,7 @@ def create_channel(): return recv, send -def list_all_channels(): +def list_all(): """Return a list of (recv, send) for all open channels.""" return [(RecvChannel(cid), SendChannel(cid)) for cid in _channels.list_all()] diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py deleted file mode 100644 index 60db7373db6dcf..00000000000000 --- a/Lib/test/test_interpreters.py +++ /dev/null @@ -1,1162 +0,0 @@ -import contextlib -import json -import os -import os.path -import sys -import threading -from textwrap import dedent -import unittest -import time - -from test import support -from test.support import import_helper -from test.support import threading_helper -from test.support import os_helper -_interpreters = import_helper.import_module('_xxsubinterpreters') -_channels = import_helper.import_module('_xxinterpchannels') -from test.support import interpreters -from test.support.interpreters import InterpreterNotFoundError - - -def _captured_script(script): - r, w = os.pipe() - indented = script.replace('\n', '\n ') - wrapped = dedent(f""" - import contextlib - with open({w}, 'w', encoding='utf-8') as spipe: - with contextlib.redirect_stdout(spipe): - {indented} - """) - return wrapped, open(r, encoding='utf-8') - - -def clean_up_interpreters(): - for interp in interpreters.list_all(): - if interp.id == 0: # main - continue - try: - interp.close() - except RuntimeError: - pass # already destroyed - - -def _run_output(interp, request, channels=None): - script, rpipe = _captured_script(request) - with rpipe: - interp.exec_sync(script, channels=channels) - return rpipe.read() - - -@contextlib.contextmanager -def _running(interp): - r, w = os.pipe() - def run(): - interp.exec_sync(dedent(f""" - # wait for "signal" - with open({r}) as rpipe: - rpipe.read() - """)) - - t = threading.Thread(target=run) - t.start() - - yield - - with open(w, 'w') as spipe: - spipe.write('done') - t.join() - - -class TestBase(unittest.TestCase): - - def pipe(self): - def ensure_closed(fd): - try: - os.close(fd) - except OSError: - pass - r, w = os.pipe() - self.addCleanup(lambda: ensure_closed(r)) - self.addCleanup(lambda: ensure_closed(w)) - return r, w - - def tearDown(self): - clean_up_interpreters() - - -class CreateTests(TestBase): - - def test_in_main(self): - interp = interpreters.create() - self.assertIsInstance(interp, interpreters.Interpreter) - self.assertIn(interp, interpreters.list_all()) - - def test_in_thread(self): - lock = threading.Lock() - interp = None - def f(): - nonlocal interp - interp = interpreters.create() - lock.acquire() - lock.release() - t = threading.Thread(target=f) - with lock: - t.start() - t.join() - self.assertIn(interp, interpreters.list_all()) - - def test_in_subinterpreter(self): - main, = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - interp = interpreters.create() - print(interp.id) - """)) - interp2 = interpreters.Interpreter(int(out)) - self.assertEqual(interpreters.list_all(), [main, interp, interp2]) - - def test_after_destroy_all(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - interp_lst = [] - for _ in range(3): - interps = interpreters.create() - interp_lst.append(interps) - # Now destroy them. - for interp in interp_lst: - interp.close() - # Finally, create another. - interp = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {interp}) - - def test_after_destroy_some(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - # Now destroy 2 of them. - interp1.close() - interp2.close() - # Finally, create another. - interp = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) - - -class GetMainTests(TestBase): - - def test_id(self): - main = interpreters.get_main() - self.assertEqual(main.id, 0) - - def test_current(self): - main = interpreters.get_main() - current = interpreters.get_current() - self.assertIs(main, current) - - def test_idempotent(self): - main1 = interpreters.get_main() - main2 = interpreters.get_main() - self.assertIs(main1, main2) - - -class GetCurrentTests(TestBase): - - def test_main(self): - main = interpreters.get_main() - current = interpreters.get_current() - self.assertEqual(current, main) - - def test_subinterpreter(self): - main = interpreters.get_main() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - cur = interpreters.get_current() - print(cur.id) - """)) - current = interpreters.Interpreter(int(out)) - self.assertEqual(current, interp) - self.assertNotEqual(current, main) - - def test_idempotent(self): - with self.subTest('main'): - cur1 = interpreters.get_current() - cur2 = interpreters.get_current() - self.assertIs(cur1, cur2) - - with self.subTest('subinterpreter'): - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - cur = interpreters.get_current() - print(id(cur)) - cur = interpreters.get_current() - print(id(cur)) - """)) - objid1, objid2 = (int(v) for v in out.splitlines()) - self.assertEqual(objid1, objid2) - - with self.subTest('per-interpreter'): - interp = interpreters.create() - out = _run_output(interp, dedent(""" - from test.support import interpreters - cur = interpreters.get_current() - print(id(cur)) - """)) - id1 = int(out) - id2 = id(interp) - self.assertNotEqual(id1, id2) - - -class ListAllTests(TestBase): - - def test_initial(self): - interps = interpreters.list_all() - self.assertEqual(1, len(interps)) - - def test_after_creating(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - - ids = [] - for interp in interpreters.list_all(): - ids.append(interp.id) - - self.assertEqual(ids, [main.id, first.id, second.id]) - - def test_after_destroying(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - first.close() - - ids = [] - for interp in interpreters.list_all(): - ids.append(interp.id) - - self.assertEqual(ids, [main.id, second.id]) - - def test_idempotent(self): - main = interpreters.get_current() - first = interpreters.create() - second = interpreters.create() - expected = [main, first, second] - - actual = interpreters.list_all() - - self.assertEqual(actual, expected) - for interp1, interp2 in zip(actual, expected): - self.assertIs(interp1, interp2) - - -class InterpreterObjectTests(TestBase): - - def test_init_int(self): - interpid = interpreters.get_current().id - interp = interpreters.Interpreter(interpid) - self.assertEqual(interp.id, interpid) - - def test_init_interpreter_id(self): - interpid = interpreters.get_current()._id - interp = interpreters.Interpreter(interpid) - self.assertEqual(interp._id, interpid) - - def test_init_unsupported(self): - actualid = interpreters.get_current().id - for interpid in [ - str(actualid), - float(actualid), - object(), - None, - '', - ]: - with self.subTest(repr(interpid)): - with self.assertRaises(TypeError): - interpreters.Interpreter(interpid) - - def test_idempotent(self): - main = interpreters.get_main() - interp = interpreters.Interpreter(main.id) - self.assertIs(interp, main) - - def test_init_does_not_exist(self): - with self.assertRaises(InterpreterNotFoundError): - interpreters.Interpreter(1_000_000) - - def test_init_bad_id(self): - with self.assertRaises(ValueError): - interpreters.Interpreter(-1) - - def test_id_type(self): - main = interpreters.get_main() - current = interpreters.get_current() - interp = interpreters.create() - self.assertIsInstance(main.id, int) - self.assertIsInstance(current.id, int) - self.assertIsInstance(interp.id, int) - - def test_id_readonly(self): - interp = interpreters.create() - with self.assertRaises(AttributeError): - interp.id = 1_000_000 - - def test_equality(self): - interp1 = interpreters.create() - interp2 = interpreters.create() - self.assertEqual(interp1, interp1) - self.assertNotEqual(interp1, interp2) - - -class TestInterpreterIsRunning(TestBase): - - def test_main(self): - main = interpreters.get_main() - self.assertTrue(main.is_running()) - - @unittest.skip('Fails on FreeBSD') - def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interp.is_running()) - - with _running(interp): - self.assertTrue(interp.is_running()) - self.assertFalse(interp.is_running()) - - def test_finished(self): - r, w = self.pipe() - interp = interpreters.create() - interp.exec_sync(f"""if True: - import os - os.write({w}, b'x') - """) - self.assertFalse(interp.is_running()) - self.assertEqual(os.read(r, 1), b'x') - - def test_from_subinterpreter(self): - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - if _interpreters.is_running({interp.id}): - print(True) - else: - print(False) - """)) - self.assertEqual(out.strip(), 'True') - - def test_already_destroyed(self): - interp = interpreters.create() - interp.close() - with self.assertRaises(InterpreterNotFoundError): - interp.is_running() - - def test_with_only_background_threads(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - DONE = b'D' - FINISHED = b'F' - - interp = interpreters.create() - interp.exec_sync(f"""if True: - import os - import threading - - def task(): - v = os.read({r_thread}, 1) - assert v == {DONE!r} - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - """) - self.assertFalse(interp.is_running()) - - os.write(w_thread, DONE) - interp.exec_sync('t.join()') - self.assertEqual(os.read(r_interp, 1), FINISHED) - - -class TestInterpreterClose(TestBase): - - def test_basic(self): - main = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp2, interp3}) - interp2.close() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp3}) - - def test_all(self): - before = set(interpreters.list_all()) - interps = set() - for _ in range(3): - interp = interpreters.create() - interps.add(interp) - self.assertEqual(set(interpreters.list_all()), before | interps) - for interp in interps: - interp.close() - self.assertEqual(set(interpreters.list_all()), before) - - def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - main.close() - - def f(): - with self.assertRaises(RuntimeError): - main.close() - - t = threading.Thread(target=f) - t.start() - t.join() - - def test_already_destroyed(self): - interp = interpreters.create() - interp.close() - with self.assertRaises(InterpreterNotFoundError): - interp.close() - - def test_from_current(self): - main, = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - from test.support import interpreters - interp = interpreters.Interpreter({interp.id}) - try: - interp.close() - except RuntimeError: - print('failed') - """)) - self.assertEqual(out.strip(), 'failed') - self.assertEqual(set(interpreters.list_all()), {main, interp}) - - def test_from_sibling(self): - main, = interpreters.list_all() - interp1 = interpreters.create() - interp2 = interpreters.create() - self.assertEqual(set(interpreters.list_all()), - {main, interp1, interp2}) - interp1.exec_sync(dedent(f""" - from test.support import interpreters - interp2 = interpreters.Interpreter({interp2.id}) - interp2.close() - interp3 = interpreters.create() - interp3.close() - """)) - self.assertEqual(set(interpreters.list_all()), {main, interp1}) - - def test_from_other_thread(self): - interp = interpreters.create() - def f(): - interp.close() - - t = threading.Thread(target=f) - t.start() - t.join() - - @unittest.skip('Fails on FreeBSD') - def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interp.close() - self.assertTrue(interp.is_running()) - - def test_subthreads_still_running(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - FINISHED = b'F' - - interp = interpreters.create() - interp.exec_sync(f"""if True: - import os - import threading - import time - - done = False - - def notify_fini(): - global done - done = True - t.join() - threading._register_atexit(notify_fini) - - def task(): - while not done: - time.sleep(0.1) - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - """) - interp.close() - - self.assertEqual(os.read(r_interp, 1), FINISHED) - - -class TestInterpreterExecSync(TestBase): - - def test_success(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - interp.exec_sync(script) - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_failure(self): - interp = interpreters.create() - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('raise Exception') - - def test_in_thread(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - def f(): - interp.exec_sync(script) - - t = threading.Thread(target=f) - t.start() - t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - @support.requires_fork() - def test_fork(self): - interp = interpreters.create() - import tempfile - with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: - file.write('') - file.flush() - - expected = 'spam spam spam spam spam' - script = dedent(f""" - import os - try: - os.fork() - except RuntimeError: - with open('{file.name}', 'w', encoding='utf-8') as out: - out.write('{expected}') - """) - interp.exec_sync(script) - - file.seek(0) - content = file.read() - self.assertEqual(content, expected) - - @unittest.skip('Fails on FreeBSD') - def test_already_running(self): - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interp.exec_sync('print("spam")') - - def test_bad_script(self): - interp = interpreters.create() - with self.assertRaises(TypeError): - interp.exec_sync(10) - - def test_bytes_for_script(self): - interp = interpreters.create() - with self.assertRaises(TypeError): - interp.exec_sync(b'print("spam")') - - def test_with_background_threads_still_running(self): - r_interp, w_interp = self.pipe() - r_thread, w_thread = self.pipe() - - RAN = b'R' - DONE = b'D' - FINISHED = b'F' - - interp = interpreters.create() - interp.exec_sync(f"""if True: - import os - import threading - - def task(): - v = os.read({r_thread}, 1) - assert v == {DONE!r} - os.write({w_interp}, {FINISHED!r}) - t = threading.Thread(target=task) - t.start() - os.write({w_interp}, {RAN!r}) - """) - interp.exec_sync(f"""if True: - os.write({w_interp}, {RAN!r}) - """) - - os.write(w_thread, DONE) - interp.exec_sync('t.join()') - self.assertEqual(os.read(r_interp, 1), RAN) - self.assertEqual(os.read(r_interp, 1), RAN) - self.assertEqual(os.read(r_interp, 1), FINISHED) - - # test_xxsubinterpreters covers the remaining - # Interpreter.exec_sync() behavior. - - -class StressTests(TestBase): - - # In these tests we generally want a lot of interpreters, - # but not so many that any test takes too long. - - @support.requires_resource('cpu') - def test_create_many_sequential(self): - alive = [] - for _ in range(100): - interp = interpreters.create() - alive.append(interp) - - @support.requires_resource('cpu') - def test_create_many_threaded(self): - alive = [] - def task(): - interp = interpreters.create() - alive.append(interp) - threads = (threading.Thread(target=task) for _ in range(200)) - with threading_helper.start_threads(threads): - pass - - -class StartupTests(TestBase): - - # We want to ensure the initial state of subinterpreters - # matches expectations. - - _subtest_count = 0 - - @contextlib.contextmanager - def subTest(self, *args): - with super().subTest(*args) as ctx: - self._subtest_count += 1 - try: - yield ctx - finally: - if self._debugged_in_subtest: - if self._subtest_count == 1: - # The first subtest adds a leading newline, so we - # compensate here by not printing a trailing newline. - print('### end subtest debug ###', end='') - else: - print('### end subtest debug ###') - self._debugged_in_subtest = False - - def debug(self, msg, *, header=None): - if header: - self._debug(f'--- {header} ---') - if msg: - if msg.endswith(os.linesep): - self._debug(msg[:-len(os.linesep)]) - else: - self._debug(msg) - self._debug('') - self._debug('------') - else: - self._debug(msg) - - _debugged = False - _debugged_in_subtest = False - def _debug(self, msg): - if not self._debugged: - print() - self._debugged = True - if self._subtest is not None: - if True: - if not self._debugged_in_subtest: - self._debugged_in_subtest = True - print('### start subtest debug ###') - print(msg) - else: - print(msg) - - def create_temp_dir(self): - import tempfile - tmp = tempfile.mkdtemp(prefix='test_interpreters_') - tmp = os.path.realpath(tmp) - self.addCleanup(os_helper.rmtree, tmp) - return tmp - - def write_script(self, *path, text): - filename = os.path.join(*path) - dirname = os.path.dirname(filename) - if dirname: - os.makedirs(dirname, exist_ok=True) - with open(filename, 'w', encoding='utf-8') as outfile: - outfile.write(dedent(text)) - return filename - - @support.requires_subprocess() - def run_python(self, argv, *, cwd=None): - # This method is inspired by - # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. - import shlex - import subprocess - if isinstance(argv, str): - argv = shlex.split(argv) - argv = [sys.executable, *argv] - try: - proc = subprocess.run( - argv, - cwd=cwd, - capture_output=True, - text=True, - ) - except Exception as exc: - self.debug(f'# cmd: {shlex.join(argv)}') - if isinstance(exc, FileNotFoundError) and not exc.filename: - if os.path.exists(argv[0]): - exists = 'exists' - else: - exists = 'does not exist' - self.debug(f'{argv[0]} {exists}') - raise # re-raise - assert proc.stderr == '' or proc.returncode != 0, proc.stderr - if proc.returncode != 0 and support.verbose: - self.debug(f'# python3 {shlex.join(argv[1:])} failed:') - self.debug(proc.stdout, header='stdout') - self.debug(proc.stderr, header='stderr') - self.assertEqual(proc.returncode, 0) - self.assertEqual(proc.stderr, '') - return proc.stdout - - def test_sys_path_0(self): - # The main interpreter's sys.path[0] should be used by subinterpreters. - script = ''' - import sys - from test.support import interpreters - - orig = sys.path[0] - - interp = interpreters.create() - interp.exec_sync(f"""if True: - import json - import sys - print(json.dumps({{ - 'main': {orig!r}, - 'sub': sys.path[0], - }}, indent=4), flush=True) - """) - ''' - # / - # pkg/ - # __init__.py - # __main__.py - # script.py - # script.py - cwd = self.create_temp_dir() - self.write_script(cwd, 'pkg', '__init__.py', text='') - self.write_script(cwd, 'pkg', '__main__.py', text=script) - self.write_script(cwd, 'pkg', 'script.py', text=script) - self.write_script(cwd, 'script.py', text=script) - - cases = [ - ('script.py', cwd), - ('-m script', cwd), - ('-m pkg', cwd), - ('-m pkg.script', cwd), - ('-c "import script"', ''), - ] - for argv, expected in cases: - with self.subTest(f'python3 {argv}'): - out = self.run_python(argv, cwd=cwd) - data = json.loads(out) - sp0_main, sp0_sub = data['main'], data['sub'] - self.assertEqual(sp0_sub, sp0_main) - self.assertEqual(sp0_sub, expected) - # XXX Also check them all with the -P cmdline flag? - - -class FinalizationTests(TestBase): - - def test_gh_109793(self): - # Make sure finalization finishes and the correct error code - # is reported, even when subinterpreters get cleaned up at the end. - import subprocess - argv = [sys.executable, '-c', '''if True: - from test.support import interpreters - interp = interpreters.create() - raise Exception - '''] - proc = subprocess.run(argv, capture_output=True, text=True) - self.assertIn('Traceback', proc.stderr) - if proc.returncode == 0 and support.verbose: - print() - print("--- cmd unexpected succeeded ---") - print(f"stdout:\n{proc.stdout}") - print(f"stderr:\n{proc.stderr}") - print("------") - self.assertEqual(proc.returncode, 1) - - -class TestIsShareable(TestBase): - - def test_default_shareables(self): - shareables = [ - # singletons - None, - # builtin objects - b'spam', - 'spam', - 10, - -10, - True, - False, - 100.0, - (), - (1, ('spam', 'eggs'), True), - ] - for obj in shareables: - with self.subTest(obj): - shareable = interpreters.is_shareable(obj) - self.assertTrue(shareable) - - def test_not_shareable(self): - class Cheese: - def __init__(self, name): - self.name = name - def __str__(self): - return self.name - - class SubBytes(bytes): - """A subclass of a shareable type.""" - - not_shareables = [ - # singletons - NotImplemented, - ..., - # builtin types and objects - type, - object, - object(), - Exception(), - # user-defined types and objects - Cheese, - Cheese('Wensleydale'), - SubBytes(b'spam'), - ] - for obj in not_shareables: - with self.subTest(repr(obj)): - self.assertFalse( - interpreters.is_shareable(obj)) - - -class TestChannels(TestBase): - - def test_create(self): - r, s = interpreters.create_channel() - self.assertIsInstance(r, interpreters.RecvChannel) - self.assertIsInstance(s, interpreters.SendChannel) - - def test_list_all(self): - self.assertEqual(interpreters.list_all_channels(), []) - created = set() - for _ in range(3): - ch = interpreters.create_channel() - created.add(ch) - after = set(interpreters.list_all_channels()) - self.assertEqual(after, created) - - def test_shareable(self): - rch, sch = interpreters.create_channel() - - self.assertTrue( - interpreters.is_shareable(rch)) - self.assertTrue( - interpreters.is_shareable(sch)) - - sch.send_nowait(rch) - sch.send_nowait(sch) - rch2 = rch.recv() - sch2 = rch.recv() - - self.assertEqual(rch2, rch) - self.assertEqual(sch2, sch) - - def test_is_closed(self): - rch, sch = interpreters.create_channel() - rbefore = rch.is_closed - sbefore = sch.is_closed - rch.close() - rafter = rch.is_closed - safter = sch.is_closed - - self.assertFalse(rbefore) - self.assertFalse(sbefore) - self.assertTrue(rafter) - self.assertTrue(safter) - - -class TestRecvChannelAttrs(TestBase): - - def test_id_type(self): - rch, _ = interpreters.create_channel() - self.assertIsInstance(rch.id, _channels.ChannelID) - - def test_custom_id(self): - rch = interpreters.RecvChannel(1) - self.assertEqual(rch.id, 1) - - with self.assertRaises(TypeError): - interpreters.RecvChannel('1') - - def test_id_readonly(self): - rch = interpreters.RecvChannel(1) - with self.assertRaises(AttributeError): - rch.id = 2 - - def test_equality(self): - ch1, _ = interpreters.create_channel() - ch2, _ = interpreters.create_channel() - self.assertEqual(ch1, ch1) - self.assertNotEqual(ch1, ch2) - - -class TestSendChannelAttrs(TestBase): - - def test_id_type(self): - _, sch = interpreters.create_channel() - self.assertIsInstance(sch.id, _channels.ChannelID) - - def test_custom_id(self): - sch = interpreters.SendChannel(1) - self.assertEqual(sch.id, 1) - - with self.assertRaises(TypeError): - interpreters.SendChannel('1') - - def test_id_readonly(self): - sch = interpreters.SendChannel(1) - with self.assertRaises(AttributeError): - sch.id = 2 - - def test_equality(self): - _, ch1 = interpreters.create_channel() - _, ch2 = interpreters.create_channel() - self.assertEqual(ch1, ch1) - self.assertNotEqual(ch1, ch2) - - -class TestSendRecv(TestBase): - - def test_send_recv_main(self): - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_same_interpreter(self): - interp = interpreters.create() - interp.exec_sync(dedent(""" - from test.support import interpreters - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv() - assert obj == orig, 'expected: obj == orig' - assert obj is not orig, 'expected: obj is not orig' - """)) - - @unittest.skip('broken (see BPO-...)') - def test_send_recv_different_interpreters(self): - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - orig1 = b'spam' - s1.send_nowait(orig1) - out = _run_output( - interpreters.create(), - dedent(f""" - obj1 = r.recv() - assert obj1 == b'spam', 'expected: obj1 == orig1' - # When going to another interpreter we get a copy. - assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' - orig2 = b'eggs' - print(id(orig2)) - s.send_nowait(orig2) - """), - channels=dict(r=r1, s=s2), - ) - obj2 = r2.recv() - - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) - - def test_send_recv_different_threads(self): - r, s = interpreters.create_channel() - - def f(): - while True: - try: - obj = r.recv() - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - s.send(obj) - t = threading.Thread(target=f) - t.start() - - orig = b'spam' - s.send(orig) - obj = r.recv() - t.join() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_nowait_main(self): - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv_nowait() - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_nowait_main_with_default(self): - r, _ = interpreters.create_channel() - obj = r.recv_nowait(None) - - self.assertIsNone(obj) - - def test_send_recv_nowait_same_interpreter(self): - interp = interpreters.create() - interp.exec_sync(dedent(""" - from test.support import interpreters - r, s = interpreters.create_channel() - orig = b'spam' - s.send_nowait(orig) - obj = r.recv_nowait() - assert obj == orig, 'expected: obj == orig' - # When going back to the same interpreter we get the same object. - assert obj is not orig, 'expected: obj is not orig' - """)) - - @unittest.skip('broken (see BPO-...)') - def test_send_recv_nowait_different_interpreters(self): - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - orig1 = b'spam' - s1.send_nowait(orig1) - out = _run_output( - interpreters.create(), - dedent(f""" - obj1 = r.recv_nowait() - assert obj1 == b'spam', 'expected: obj1 == orig1' - # When going to another interpreter we get a copy. - assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' - orig2 = b'eggs' - print(id(orig2)) - s.send_nowait(orig2) - """), - channels=dict(r=r1, s=s2), - ) - obj2 = r2.recv_nowait() - - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) - - def test_recv_timeout(self): - r, _ = interpreters.create_channel() - with self.assertRaises(TimeoutError): - r.recv(timeout=1) - - def test_recv_channel_does_not_exist(self): - ch = interpreters.RecvChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.recv() - - def test_send_channel_does_not_exist(self): - ch = interpreters.SendChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.send(b'spam') - - def test_recv_nowait_channel_does_not_exist(self): - ch = interpreters.RecvChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.recv_nowait() - - def test_send_nowait_channel_does_not_exist(self): - ch = interpreters.SendChannel(1_000_000) - with self.assertRaises(interpreters.ChannelNotFoundError): - ch.send_nowait(b'spam') - - def test_recv_nowait_empty(self): - ch, _ = interpreters.create_channel() - with self.assertRaises(interpreters.ChannelEmptyError): - ch.recv_nowait() - - def test_recv_nowait_default(self): - default = object() - rch, sch = interpreters.create_channel() - obj1 = rch.recv_nowait(default) - sch.send_nowait(None) - sch.send_nowait(1) - sch.send_nowait(b'spam') - sch.send_nowait(b'eggs') - obj2 = rch.recv_nowait(default) - obj3 = rch.recv_nowait(default) - obj4 = rch.recv_nowait() - obj5 = rch.recv_nowait(default) - obj6 = rch.recv_nowait(default) - - self.assertIs(obj1, default) - self.assertIs(obj2, None) - self.assertEqual(obj3, 1) - self.assertEqual(obj4, b'spam') - self.assertEqual(obj5, b'eggs') - self.assertIs(obj6, default) - - def test_send_buffer(self): - buf = bytearray(b'spamspamspam') - obj = None - rch, sch = interpreters.create_channel() - - def f(): - nonlocal obj - while True: - try: - obj = rch.recv() - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - t = threading.Thread(target=f) - t.start() - - sch.send_buffer(buf) - t.join() - - self.assertIsNot(obj, buf) - self.assertIsInstance(obj, memoryview) - self.assertEqual(obj, buf) - - buf[4:8] = b'eggs' - self.assertEqual(obj, buf) - obj[4:8] = b'ham.' - self.assertEqual(obj, buf) - - def test_send_buffer_nowait(self): - buf = bytearray(b'spamspamspam') - rch, sch = interpreters.create_channel() - sch.send_buffer_nowait(buf) - obj = rch.recv() - - self.assertIsNot(obj, buf) - self.assertIsInstance(obj, memoryview) - self.assertEqual(obj, buf) - - buf[4:8] = b'eggs' - self.assertEqual(obj, buf) - obj[4:8] = b'ham.' - self.assertEqual(obj, buf) diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_interpreters/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_interpreters/__main__.py b/Lib/test/test_interpreters/__main__.py new file mode 100644 index 00000000000000..8641229877b2be --- /dev/null +++ b/Lib/test/test_interpreters/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +nittest.main() diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py new file mode 100644 index 00000000000000..d94a03bccf2e97 --- /dev/null +++ b/Lib/test/test_interpreters/test_api.py @@ -0,0 +1,596 @@ +import os +import threading +from textwrap import dedent +import unittest + +from test import support +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from test.support import interpreters +from test.support.interpreters import InterpreterNotFoundError +from .utils import _captured_script, _run_output, _running, TestBase + + +class CreateTests(TestBase): + + def test_in_main(self): + interp = interpreters.create() + self.assertIsInstance(interp, interpreters.Interpreter) + self.assertIn(interp, interpreters.list_all()) + + def test_in_thread(self): + lock = threading.Lock() + interp = None + def f(): + nonlocal interp + interp = interpreters.create() + lock.acquire() + lock.release() + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(interp, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + interp = interpreters.create() + print(interp.id) + """)) + interp2 = interpreters.Interpreter(int(out)) + self.assertEqual(interpreters.list_all(), [main, interp, interp2]) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp_lst = [] + for _ in range(3): + interps = interpreters.create() + interp_lst.append(interps) + # Now destroy them. + for interp in interp_lst: + interp.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + # Now destroy 2 of them. + interp1.close() + interp2.close() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {interp3, interp}) + + +class GetMainTests(TestBase): + + def test_id(self): + main = interpreters.get_main() + self.assertEqual(main.id, 0) + + def test_current(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertIs(main, current) + + def test_idempotent(self): + main1 = interpreters.get_main() + main2 = interpreters.get_main() + self.assertIs(main1, main2) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + current = interpreters.get_current() + self.assertEqual(current, main) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(cur.id) + """)) + current = interpreters.Interpreter(int(out)) + self.assertEqual(current, interp) + self.assertNotEqual(current, main) + + def test_idempotent(self): + with self.subTest('main'): + cur1 = interpreters.get_current() + cur2 = interpreters.get_current() + self.assertIs(cur1, cur2) + + with self.subTest('subinterpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + cur = interpreters.get_current() + print(id(cur)) + """)) + objid1, objid2 = (int(v) for v in out.splitlines()) + self.assertEqual(objid1, objid2) + + with self.subTest('per-interpreter'): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + from test.support import interpreters + cur = interpreters.get_current() + print(id(cur)) + """)) + id1 = int(out) + id2 = id(interp) + self.assertNotEqual(id1, id2) + + +class ListAllTests(TestBase): + + def test_initial(self): + interps = interpreters.list_all() + self.assertEqual(1, len(interps)) + + def test_after_creating(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, first.id, second.id]) + + def test_after_destroying(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + first.close() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, second.id]) + + def test_idempotent(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + expected = [main, first, second] + + actual = interpreters.list_all() + + self.assertEqual(actual, expected) + for interp1, interp2 in zip(actual, expected): + self.assertIs(interp1, interp2) + + +class InterpreterObjectTests(TestBase): + + def test_init_int(self): + interpid = interpreters.get_current().id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp.id, interpid) + + def test_init_interpreter_id(self): + interpid = interpreters.get_current()._id + interp = interpreters.Interpreter(interpid) + self.assertEqual(interp._id, interpid) + + def test_init_unsupported(self): + actualid = interpreters.get_current().id + for interpid in [ + str(actualid), + float(actualid), + object(), + None, + '', + ]: + with self.subTest(repr(interpid)): + with self.assertRaises(TypeError): + interpreters.Interpreter(interpid) + + def test_idempotent(self): + main = interpreters.get_main() + interp = interpreters.Interpreter(main.id) + self.assertIs(interp, main) + + def test_init_does_not_exist(self): + with self.assertRaises(InterpreterNotFoundError): + interpreters.Interpreter(1_000_000) + + def test_init_bad_id(self): + with self.assertRaises(ValueError): + interpreters.Interpreter(-1) + + def test_id_type(self): + main = interpreters.get_main() + current = interpreters.get_current() + interp = interpreters.create() + self.assertIsInstance(main.id, int) + self.assertIsInstance(current.id, int) + self.assertIsInstance(interp.id, int) + + def test_id_readonly(self): + interp = interpreters.create() + with self.assertRaises(AttributeError): + interp.id = 1_000_000 + + def test_hashable(self): + interp = interpreters.create() + expected = hash(interp.id) + actual = hash(interp) + self.assertEqual(actual, expected) + + def test_equality(self): + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(interp1, interp1) + self.assertNotEqual(interp1, interp2) + + +class TestInterpreterIsRunning(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(main.is_running()) + + @unittest.skip('Fails on FreeBSD') + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interp.is_running()) + + with _running(interp): + self.assertTrue(interp.is_running()) + self.assertFalse(interp.is_running()) + + def test_finished(self): + r, w = self.pipe() + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + os.write({w}, b'x') + """) + self.assertFalse(interp.is_running()) + self.assertEqual(os.read(r, 1), b'x') + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + if _interpreters.is_running({interp.id}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.is_running() + + def test_with_only_background_threads(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + self.assertFalse(interp.is_running()) + + os.write(w_thread, DONE) + interp.exec_sync('t.join()') + self.assertEqual(os.read(r_interp, 1), FINISHED) + + +class TestInterpreterClose(TestBase): + + def test_basic(self): + main = interpreters.get_main() + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2, interp3}) + interp2.close() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp3}) + + def test_all(self): + before = set(interpreters.list_all()) + interps = set() + for _ in range(3): + interp = interpreters.create() + interps.add(interp) + self.assertEqual(set(interpreters.list_all()), before | interps) + for interp in interps: + interp.close() + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + main.close() + + def f(): + with self.assertRaises(RuntimeError): + main.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + interp = interpreters.create() + interp.close() + with self.assertRaises(InterpreterNotFoundError): + interp.close() + + def test_from_current(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + from test.support import interpreters + interp = interpreters.Interpreter({interp.id}) + try: + interp.close() + except RuntimeError: + print('failed') + """)) + self.assertEqual(out.strip(), 'failed') + self.assertEqual(set(interpreters.list_all()), {main, interp}) + + def test_from_sibling(self): + main, = interpreters.list_all() + interp1 = interpreters.create() + interp2 = interpreters.create() + self.assertEqual(set(interpreters.list_all()), + {main, interp1, interp2}) + interp1.exec_sync(dedent(f""" + from test.support import interpreters + interp2 = interpreters.Interpreter({interp2.id}) + interp2.close() + interp3 = interpreters.create() + interp3.close() + """)) + self.assertEqual(set(interpreters.list_all()), {main, interp1}) + + def test_from_other_thread(self): + interp = interpreters.create() + def f(): + interp.close() + + t = threading.Thread(target=f) + t.start() + t.join() + + @unittest.skip('Fails on FreeBSD') + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.close() + self.assertTrue(interp.is_running()) + + def test_subthreads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + + +class TestInterpreterExecSync(TestBase): + + def test_success(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + interp.exec_sync(script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_failure(self): + interp = interpreters.create() + with self.assertRaises(interpreters.ExecFailure): + interp.exec_sync('raise Exception') + + def test_in_thread(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + interp.exec_sync(script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + @support.requires_fork() + def test_fork(self): + interp = interpreters.create() + import tempfile + with tempfile.NamedTemporaryFile('w+', encoding='utf-8') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w', encoding='utf-8') as out: + out.write('{expected}') + """) + interp.exec_sync(script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + @unittest.skip('Fails on FreeBSD') + def test_already_running(self): + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.exec_sync('print("spam")') + + def test_bad_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec_sync(10) + + def test_bytes_for_script(self): + interp = interpreters.create() + with self.assertRaises(TypeError): + interp.exec_sync(b'print("spam")') + + def test_with_background_threads_still_running(self): + r_interp, w_interp = self.pipe() + r_thread, w_thread = self.pipe() + + RAN = b'R' + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + os.write({w_interp}, {RAN!r}) + """) + interp.exec_sync(f"""if True: + os.write({w_interp}, {RAN!r}) + """) + + os.write(w_thread, DONE) + interp.exec_sync('t.join()') + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), FINISHED) + + # test_xxsubinterpreters covers the remaining + # Interpreter.exec_sync() behavior. + + +class TestIsShareable(TestBase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + True, + False, + 100.0, + (), + (1, ('spam', 'eggs'), True), + ] + for obj in shareables: + with self.subTest(obj): + shareable = interpreters.is_shareable(obj) + self.assertTrue(shareable) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_channel.py b/Lib/test/test_interpreters/test_channel.py new file mode 100644 index 00000000000000..f38237e0b153b9 --- /dev/null +++ b/Lib/test/test_interpreters/test_channel.py @@ -0,0 +1,328 @@ +import threading +from textwrap import dedent +import unittest +import time + +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +_channels = import_helper.import_module('_xxinterpchannels') +from test.support import interpreters +import test.support.interpreters.channel as channels +from .utils import _run_output, TestBase + + +class TestChannels(TestBase): + + def test_create(self): + r, s = channels.create() + self.assertIsInstance(r, channels.RecvChannel) + self.assertIsInstance(s, channels.SendChannel) + + def test_list_all(self): + self.assertEqual(channels.list_all(), []) + created = set() + for _ in range(3): + ch = channels.create() + created.add(ch) + after = set(channels.list_all()) + self.assertEqual(after, created) + + def test_shareable(self): + rch, sch = channels.create() + + self.assertTrue( + interpreters.is_shareable(rch)) + self.assertTrue( + interpreters.is_shareable(sch)) + + sch.send_nowait(rch) + sch.send_nowait(sch) + rch2 = rch.recv() + sch2 = rch.recv() + + self.assertEqual(rch2, rch) + self.assertEqual(sch2, sch) + + def test_is_closed(self): + rch, sch = channels.create() + rbefore = rch.is_closed + sbefore = sch.is_closed + rch.close() + rafter = rch.is_closed + safter = sch.is_closed + + self.assertFalse(rbefore) + self.assertFalse(sbefore) + self.assertTrue(rafter) + self.assertTrue(safter) + + +class TestRecvChannelAttrs(TestBase): + + def test_id_type(self): + rch, _ = channels.create() + self.assertIsInstance(rch.id, _channels.ChannelID) + + def test_custom_id(self): + rch = channels.RecvChannel(1) + self.assertEqual(rch.id, 1) + + with self.assertRaises(TypeError): + channels.RecvChannel('1') + + def test_id_readonly(self): + rch = channels.RecvChannel(1) + with self.assertRaises(AttributeError): + rch.id = 2 + + def test_equality(self): + ch1, _ = channels.create() + ch2, _ = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + +class TestSendChannelAttrs(TestBase): + + def test_id_type(self): + _, sch = channels.create() + self.assertIsInstance(sch.id, _channels.ChannelID) + + def test_custom_id(self): + sch = channels.SendChannel(1) + self.assertEqual(sch.id, 1) + + with self.assertRaises(TypeError): + channels.SendChannel('1') + + def test_id_readonly(self): + sch = channels.SendChannel(1) + with self.assertRaises(AttributeError): + sch.id = 2 + + def test_equality(self): + _, ch1 = channels.create() + _, ch2 = channels.create() + self.assertEqual(ch1, ch1) + self.assertNotEqual(ch1, ch2) + + +class TestSendRecv(TestBase): + + def test_send_recv_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + import test.support.interpreters.channel as channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_send_recv_different_threads(self): + r, s = channels.create() + + def f(): + while True: + try: + obj = r.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + s.send(orig) + obj = r.recv() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main(self): + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_main_with_default(self): + r, _ = channels.create() + obj = r.recv_nowait(None) + + self.assertIsNone(obj) + + def test_send_recv_nowait_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + import test.support.interpreters.channel as channels + r, s = channels.create() + orig = b'spam' + s.send_nowait(orig) + obj = r.recv_nowait() + assert obj == orig, 'expected: obj == orig' + # When going back to the same interpreter we get the same object. + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.skip('broken (see BPO-...)') + def test_send_recv_nowait_different_interpreters(self): + r1, s1 = channels.create() + r2, s2 = channels.create() + orig1 = b'spam' + s1.send_nowait(orig1) + out = _run_output( + interpreters.create(), + dedent(f""" + obj1 = r.recv_nowait() + assert obj1 == b'spam', 'expected: obj1 == orig1' + # When going to another interpreter we get a copy. + assert id(obj1) != {id(orig1)}, 'expected: obj1 is not orig1' + orig2 = b'eggs' + print(id(orig2)) + s.send_nowait(orig2) + """), + channels=dict(r=r1, s=s2), + ) + obj2 = r2.recv_nowait() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_recv_timeout(self): + r, _ = channels.create() + with self.assertRaises(TimeoutError): + r.recv(timeout=1) + + def test_recv_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv() + + def test_send_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send(b'spam') + + def test_recv_nowait_channel_does_not_exist(self): + ch = channels.RecvChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.recv_nowait() + + def test_send_nowait_channel_does_not_exist(self): + ch = channels.SendChannel(1_000_000) + with self.assertRaises(channels.ChannelNotFoundError): + ch.send_nowait(b'spam') + + def test_recv_nowait_empty(self): + ch, _ = channels.create() + with self.assertRaises(channels.ChannelEmptyError): + ch.recv_nowait() + + def test_recv_nowait_default(self): + default = object() + rch, sch = channels.create() + obj1 = rch.recv_nowait(default) + sch.send_nowait(None) + sch.send_nowait(1) + sch.send_nowait(b'spam') + sch.send_nowait(b'eggs') + obj2 = rch.recv_nowait(default) + obj3 = rch.recv_nowait(default) + obj4 = rch.recv_nowait() + obj5 = rch.recv_nowait(default) + obj6 = rch.recv_nowait(default) + + self.assertIs(obj1, default) + self.assertIs(obj2, None) + self.assertEqual(obj3, 1) + self.assertEqual(obj4, b'spam') + self.assertEqual(obj5, b'eggs') + self.assertIs(obj6, default) + + def test_send_buffer(self): + buf = bytearray(b'spamspamspam') + obj = None + rch, sch = channels.create() + + def f(): + nonlocal obj + while True: + try: + obj = rch.recv() + break + except channels.ChannelEmptyError: + time.sleep(0.1) + t = threading.Thread(target=f) + t.start() + + sch.send_buffer(buf) + t.join() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + def test_send_buffer_nowait(self): + buf = bytearray(b'spamspamspam') + rch, sch = channels.create() + sch.send_buffer_nowait(buf) + obj = rch.recv() + + self.assertIsNot(obj, buf) + self.assertIsInstance(obj, memoryview) + self.assertEqual(obj, buf) + + buf[4:8] = b'eggs' + self.assertEqual(obj, buf) + obj[4:8] = b'ham.' + self.assertEqual(obj, buf) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py new file mode 100644 index 00000000000000..c2917d839904f9 --- /dev/null +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -0,0 +1,189 @@ +import contextlib +import json +import os +import os.path +import sys +from textwrap import dedent +import unittest + +from test import support +from test.support import import_helper +from test.support import os_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from .utils import TestBase + + +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr + if proc.returncode != 0 and support.verbose: + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + script = ''' + import sys + from test.support import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.exec_sync(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + ''' + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), + ] + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + +class FinalizationTests(TestBase): + + def test_gh_109793(self): + # Make sure finalization finishes and the correct error code + # is reported, even when subinterpreters get cleaned up at the end. + import subprocess + argv = [sys.executable, '-c', '''if True: + from test.support import interpreters + interp = interpreters.create() + raise Exception + '''] + proc = subprocess.run(argv, capture_output=True, text=True) + self.assertIn('Traceback', proc.stderr) + if proc.returncode == 0 and support.verbose: + print() + print("--- cmd unexpected succeeded ---") + print(f"stdout:\n{proc.stdout}") + print(f"stderr:\n{proc.stderr}") + print("------") + self.assertEqual(proc.returncode, 1) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py new file mode 100644 index 00000000000000..3cc570b3bf7128 --- /dev/null +++ b/Lib/test/test_interpreters/test_stress.py @@ -0,0 +1,38 @@ +import threading +import unittest + +from test import support +from test.support import import_helper +from test.support import threading_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxsubinterpreters') +from test.support import interpreters +from .utils import TestBase + + +class StressTests(TestBase): + + # In these tests we generally want a lot of interpreters, + # but not so many that any test takes too long. + + @support.requires_resource('cpu') + def test_create_many_sequential(self): + alive = [] + for _ in range(100): + interp = interpreters.create() + alive.append(interp) + + @support.requires_resource('cpu') + def test_create_many_threaded(self): + alive = [] + def task(): + interp = interpreters.create() + alive.append(interp) + threads = (threading.Thread(target=task) for _ in range(200)) + with threading_helper.start_threads(threads): + pass + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py new file mode 100644 index 00000000000000..623c8737b79831 --- /dev/null +++ b/Lib/test/test_interpreters/utils.py @@ -0,0 +1,73 @@ +import contextlib +import os +import threading +from textwrap import dedent +import unittest + +from test.support import interpreters + + +def _captured_script(script): + r, w = os.pipe() + indented = script.replace('\n', '\n ') + wrapped = dedent(f""" + import contextlib + with open({w}, 'w', encoding='utf-8') as spipe: + with contextlib.redirect_stdout(spipe): + {indented} + """) + return wrapped, open(r, encoding='utf-8') + + +def clean_up_interpreters(): + for interp in interpreters.list_all(): + if interp.id == 0: # main + continue + try: + interp.close() + except RuntimeError: + pass # already destroyed + + +def _run_output(interp, request, channels=None): + script, rpipe = _captured_script(request) + with rpipe: + interp.exec_sync(script, channels=channels) + return rpipe.read() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interp.exec_sync(dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + +class TestBase(unittest.TestCase): + + def pipe(self): + def ensure_closed(fd): + try: + os.close(fd) + except OSError: + pass + r, w = os.pipe() + self.addCleanup(lambda: ensure_closed(r)) + self.addCleanup(lambda: ensure_closed(w)) + return r, w + + def tearDown(self): + clean_up_interpreters() From 8ca303fe7fe79bdb161530ed4832625efb328113 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2023 16:51:32 -0700 Subject: [PATCH 11/26] Add interpreters.Queue. --- Lib/test/support/interpreters/__init__.py | 7 +- Lib/test/support/interpreters/queue.py | 156 +++++++++++++++ Lib/test/test_interpreters/test_queue.py | 233 ++++++++++++++++++++++ 3 files changed, 394 insertions(+), 2 deletions(-) create mode 100644 Lib/test/support/interpreters/queue.py create mode 100644 Lib/test/test_interpreters/test_queue.py diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 86b625640a439f..f7a7b7053eeec6 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -8,14 +8,17 @@ InterpreterError, InterpreterNotFoundError, is_shareable, ) -from .channel import create as create_channel +from .queue import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, +) __all__ = [ 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', 'Interpreter', 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', - 'create_channel', + 'create_queue', 'QueueEmpty', 'QueueFull', ] diff --git a/Lib/test/support/interpreters/queue.py b/Lib/test/support/interpreters/queue.py new file mode 100644 index 00000000000000..ed6b0d551dd890 --- /dev/null +++ b/Lib/test/support/interpreters/queue.py @@ -0,0 +1,156 @@ +"""Cross-interpreter Queues High Level Module.""" + +import queue +import time +import weakref +import _xxinterpchannels as _channels +import _xxinterpchannels as _queues + +# aliases: +from _xxinterpchannels import ( + ChannelError as QueueError, + ChannelNotFoundError as QueueNotFoundError, +) + +__all__ = [ + 'create', 'list_all', + 'Queue', + 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', +] + + +def create(maxsize=0): + """Return a new cross-interpreter queue. + + The queue may be used to pass data safely between interpreters. + """ + # XXX honor maxsize + qid = _queues.create() + return Queue._with_maxsize(qid, maxsize) + + +def list_all(): + """Return a list of all open queues.""" + return [Queue(qid) + for qid in _queues.list_all()] + + +class QueueEmpty(queue.Empty): + """Raised from get_nowait() when the queue is empty. + + It is also raised from get() if it times out. + """ + + +class QueueFull(queue.Full): + """Raised from put_nowait() when the queue is full. + + It is also raised from put() if it times out. + """ + + +_known_queues = weakref.WeakValueDictionary() + +class Queue: + """A cross-interpreter queue.""" + + @classmethod + def _with_maxsize(cls, id, maxsize): + if not isinstance(maxsize, int): + raise TypeError(f'maxsize must be an int, got {maxsize!r}') + elif maxsize < 0: + maxsize = 0 + else: + maxsize = int(maxsize) + self = cls(id) + self._maxsize = maxsize + return self + + def __new__(cls, id, /): + # There is only one instance for any given ID. + if isinstance(id, int): + id = _channels._channel_id(id, force=False) + elif not isinstance(id, _channels.ChannelID): + raise TypeError(f'id must be an int, got {id!r}') + key = int(id) + try: + self = _known_queues[key] + except KeyError: + self = super().__new__(cls) + self._id = id + self._maxsize = 0 + _known_queues[key] = self + return self + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + @property + def id(self): + return int(self._id) + + @property + def maxsize(self): + return self._maxsize + + @property + def _info(self): + return _channels.get_info(self._id) + + def empty(self): + return self._info.count == 0 + + def full(self): + if self._maxsize <= 0: + return False + return self._info.count >= self._maxsize + + def qsize(self): + return self._info.count + + def put(self, obj, timeout=None): + # XXX block if full + _channels.send(self._id, obj, blocking=False) + + def put_nowait(self, obj): + # XXX raise QueueFull if full + return _channels.send(self._id, obj, blocking=False) + + def get(self, timeout=None, *, + _sentinel=object(), + _delay=10 / 1000, # 10 milliseconds + ): + """Return the next object from the queue. + + This blocks while the queue is empty. + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + obj = _channels.recv(self._id, _sentinel) + while obj is _sentinel: + time.sleep(_delay) + if timeout is not None and time.time() >= end: + raise QueueEmpty + obj = _channels.recv(self._id, _sentinel) + return obj + + def get_nowait(self, *, _sentinel=object()): + """Return the next object from the channel. + + If the queue is empty then raise QueueEmpty. Otherwise this + is the same as get(). + """ + obj = _channels.recv(self._id, _sentinel) + if obj is _sentinel: + raise QueueEmpty + return obj + + +# XXX add this: +#_channels._register_queue_type(Queue) diff --git a/Lib/test/test_interpreters/test_queue.py b/Lib/test/test_interpreters/test_queue.py new file mode 100644 index 00000000000000..cf6ac5f0990d2f --- /dev/null +++ b/Lib/test/test_interpreters/test_queue.py @@ -0,0 +1,233 @@ +import threading +from textwrap import dedent +import unittest +import time + +from test.support import import_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_xxinterpchannels') +#import_helper.import_module('_xxinterpqueues') +from test.support import interpreters +import test.support.interpreters.queue as queues +from .utils import _run_output, TestBase + + +class QueueTests(TestBase): + + def test_create(self): + with self.subTest('vanilla'): + queue = queues.create() + self.assertEqual(queue.maxsize, 0) + + with self.subTest('small maxsize'): + queue = queues.create(3) + self.assertEqual(queue.maxsize, 3) + + with self.subTest('big maxsize'): + queue = queues.create(100) + self.assertEqual(queue.maxsize, 100) + + with self.subTest('no maxsize'): + queue = queues.create(0) + self.assertEqual(queue.maxsize, 0) + + with self.subTest('negative maxsize'): + queue = queues.create(-1) + self.assertEqual(queue.maxsize, 0) + + with self.subTest('bad maxsize'): + with self.assertRaises(TypeError): + queues.create('1') + + @unittest.expectedFailure + def test_shareable(self): + queue1 = queues.create() + queue2 = queues.create() + queue1.put(queue2) + queue3 = queue1.get() + self.assertIs(queue3, queue1) + + def test_id_type(self): + queue = queues.create() + self.assertIsInstance(queue.id, int) + + def test_custom_id(self): + with self.assertRaises(queues.QueueNotFoundError): + queues.Queue(1_000_000) + + def test_id_readonly(self): + queue = queues.create() + with self.assertRaises(AttributeError): + queue.id = 1_000_000 + + def test_maxsize_readonly(self): + queue = queues.create(10) + with self.assertRaises(AttributeError): + queue.maxsize = 1_000_000 + + def test_hashable(self): + queue = queues.create() + expected = hash(queue.id) + actual = hash(queue) + self.assertEqual(actual, expected) + + def test_equality(self): + queue1 = queues.create() + queue2 = queues.create() + self.assertEqual(queue1, queue1) + self.assertNotEqual(queue1, queue2) + + +class TestQueueOps(TestBase): + + def test_empty(self): + queue = queues.create() + before = queue.empty() + queue.put(None) + during = queue.empty() + queue.get() + after = queue.empty() + + self.assertIs(before, True) + self.assertIs(during, False) + self.assertIs(after, True) + + def test_full(self): + expected = [False, False, False, True, False, False, False] + actual = [] + queue = queues.create(3) + for _ in range(3): + actual.append(queue.full()) + queue.put(None) + actual.append(queue.full()) + for _ in range(3): + queue.get() + actual.append(queue.full()) + + self.assertEqual(actual, expected) + + def test_qsize(self): + expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0] + actual = [] + queue = queues.create() + for _ in range(3): + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + for _ in range(3): + queue.get() + actual.append(queue.qsize()) + queue.put(None) + actual.append(queue.qsize()) + queue.get() + actual.append(queue.qsize()) + + self.assertEqual(actual, expected) + + def test_put_get_main(self): + expected = list(range(20)) + queue = queues.create() + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] + + self.assertEqual(actual, expected) + + @unittest.expectedFailure + def test_put_timeout(self): + queue = queues.create(2) + queue.put(None) + queue.put(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1) + queue.get() + queue.put(None) + + @unittest.expectedFailure + def test_put_nowait(self): + queue = queues.create(2) + queue.put_nowait(None) + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + queue.get() + queue.put_nowait(None) + + def test_get_timeout(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get(timeout=0.1) + + def test_get_nowait(self): + queue = queues.create() + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + def test_put_get_same_interpreter(self): + interp = interpreters.create() + interp.exec_sync(dedent(""" + import test.support.interpreters.queue as queues + queue = queues.create() + orig = b'spam' + queue.put(orig) + obj = queue.get() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) + + @unittest.expectedFailure + def test_put_get_different_interpreters(self): + queue1 = queues.create() + queue2 = queues.create() + obj1 = b'spam' + queue1.put(obj1) + out = _run_output( + interpreters.create(), + dedent(f""" + import test.support.interpreters.queue as queues + queue1 = queues.Queue({queue1.id}) + queue2 = queues.Queue({queue2.id}) + obj = queue1.get() + assert obj == b'spam', 'expected: obj == obj1' + # When going to another interpreter we get a copy. + assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' + obj2 = b'eggs' + print(id(obj2)) + queue2.put(obj2) + """)) + obj2 = queue2.get() + + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) + + def test_put_get_different_threads(self): + queue1 = queues.create() + queue2 = queues.create() + + def f(): + while True: + try: + obj = queue1.get(timeout=0.1) + break + except queues.QueueEmpty: + continue + queue2.put(obj) + t = threading.Thread(target=f) + t.start() + + orig = b'spam' + queue1.put(orig) + obj = queue2.get() + t.join() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() From 8b9737347f1a8c638ba646271d8b9b51c47bed83 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 14:35:22 -0700 Subject: [PATCH 12/26] Add memoryview XID with _xxsubinterpreters import. --- Modules/_xxinterpchannelsmodule.c | 157 ----------------------- Modules/_xxsubinterpretersmodule.c | 197 +++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+), 157 deletions(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 1c9ae3b87adf7c..97729ec269cb62 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -8,7 +8,6 @@ #include "Python.h" #include "interpreteridobject.h" #include "pycore_crossinterp.h" // struct _xid -#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_interp.h" // _PyInterpreterState_LookUpID() #ifdef MS_WINDOWS @@ -263,136 +262,6 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) } -/* Cross-interpreter Buffer Views *******************************************/ - -// XXX Release when the original interpreter is destroyed. - -typedef struct { - PyObject_HEAD - Py_buffer *view; - int64_t interpid; -} XIBufferViewObject; - -static PyObject * -xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) -{ - assert(data->data != NULL); - assert(data->obj == NULL); - assert(data->interpid >= 0); - XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); - if (self == NULL) { - return NULL; - } - PyObject_Init((PyObject *)self, cls); - self->view = (Py_buffer *)data->data; - self->interpid = data->interpid; - return (PyObject *)self; -} - -static void -xibufferview_dealloc(XIBufferViewObject *self) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid); - /* If the interpreter is no longer alive then we have problems, - since other objects may be using the buffer still. */ - assert(interp != NULL); - - if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { - // XXX Emit a warning? - PyErr_Clear(); - } - - PyTypeObject *tp = Py_TYPE(self); - tp->tp_free(self); - /* "Instances of heap-allocated types hold a reference to their type." - * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol - * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse - */ - // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, - // like we do for _abc._abc_data? - Py_DECREF(tp); -} - -static int -xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) -{ - /* Only PyMemoryView_FromObject() should ever call this, - via _memoryview_from_xid() below. */ - *view = *self->view; - view->obj = (PyObject *)self; - // XXX Should we leave it alone? - view->internal = NULL; - return 0; -} - -static PyType_Slot XIBufferViewType_slots[] = { - {Py_tp_dealloc, (destructor)xibufferview_dealloc}, - {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, - // We don't bother with Py_bf_releasebuffer since we don't need it. - {0, NULL}, -}; - -static PyType_Spec XIBufferViewType_spec = { - .name = MODULE_NAME ".CrossInterpreterBufferView", - .basicsize = sizeof(XIBufferViewObject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), - .slots = XIBufferViewType_slots, -}; - - -/* extra XID types **********************************************************/ - -static PyTypeObject * _get_current_xibufferview_type(void); - -static PyObject * -_memoryview_from_xid(_PyCrossInterpreterData *data) -{ - PyTypeObject *cls = _get_current_xibufferview_type(); - if (cls == NULL) { - return NULL; - } - PyObject *obj = xibufferview_from_xid(cls, data); - if (obj == NULL) { - return NULL; - } - return PyMemoryView_FromObject(obj); -} - -static int -_memoryview_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); - if (view == NULL) { - return -1; - } - if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { - PyMem_RawFree(view); - return -1; - } - _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, - _memoryview_from_xid); - return 0; -} - -static int -register_builtin_xid_types(struct xid_class_registry *classes) -{ - PyTypeObject *cls; - crossinterpdatafunc func; - - // builtin memoryview - cls = &PyMemoryView_Type; - func = _memoryview_shared; - if (register_xid_class(cls, func, classes)) { - return -1; - } - - return 0; -} - - /* module state *************************************************************/ typedef struct { @@ -405,7 +274,6 @@ typedef struct { /* heap types */ PyTypeObject *ChannelInfoType; PyTypeObject *ChannelIDType; - PyTypeObject *XIBufferViewType; /* exceptions */ PyObject *ChannelError; @@ -449,7 +317,6 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) /* heap types */ Py_VISIT(state->ChannelInfoType); Py_VISIT(state->ChannelIDType); - Py_VISIT(state->XIBufferViewType); /* exceptions */ Py_VISIT(state->ChannelError); @@ -474,7 +341,6 @@ clear_module_state(module_state *state) (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); } Py_CLEAR(state->ChannelIDType); - Py_CLEAR(state->XIBufferViewType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -487,17 +353,6 @@ clear_module_state(module_state *state) } -static PyTypeObject * -_get_current_xibufferview_type(void) -{ - module_state *state = _get_current_module_state(); - if (state == NULL) { - return NULL; - } - return state->XIBufferViewType; -} - - /* channel-specific code ****************************************************/ #define CHANNEL_SEND 1 @@ -3463,18 +3318,6 @@ module_exec(PyObject *mod) goto error; } - // XIBufferView - state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL, - xid_classes); - if (state->XIBufferViewType == NULL) { - goto error; - } - - // Register external types. - if (register_builtin_xid_types(xid_classes) < 0) { - goto error; - } - /* Make sure chnnels drop objects owned by this interpreter. */ PyInterpreterState *interp = _get_current_interp(); PyUnstable_AtExit(interp, clear_interpreter, (void *)interp); diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 5954d47e2555ec..8205fa78820581 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -12,6 +12,7 @@ #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() @@ -121,11 +122,169 @@ get_interpid_obj(PyInterpreterState *interp) return interpid_to_pylong(id); } +static PyObject * +_get_current_module(void) +{ + PyObject *name = PyUnicode_FromString(MODULE_NAME); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + + +/* Cross-interpreter Buffer Views *******************************************/ + +// XXX Release when the original interpreter is destroyed. + +typedef struct { + PyObject_HEAD + Py_buffer *view; + int64_t interpid; +} XIBufferViewObject; + +static PyObject * +xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) +{ + assert(data->data != NULL); + assert(data->obj == NULL); + assert(data->interpid >= 0); + XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); + if (self == NULL) { + return NULL; + } + PyObject_Init((PyObject *)self, cls); + self->view = (Py_buffer *)data->data; + self->interpid = data->interpid; + return (PyObject *)self; +} + +static void +xibufferview_dealloc(XIBufferViewObject *self) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid); + /* If the interpreter is no longer alive then we have problems, + since other objects may be using the buffer still. */ + assert(interp != NULL); + + if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) { + // XXX Emit a warning? + PyErr_Clear(); + } + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + /* "Instances of heap-allocated types hold a reference to their type." + * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol + * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse + */ + // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, + // like we do for _abc._abc_data? + Py_DECREF(tp); +} + +static int +xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags) +{ + /* Only PyMemoryView_FromObject() should ever call this, + via _memoryview_from_xid() below. */ + *view = *self->view; + view->obj = (PyObject *)self; + // XXX Should we leave it alone? + view->internal = NULL; + return 0; +} + +static PyType_Slot XIBufferViewType_slots[] = { + {Py_tp_dealloc, (destructor)xibufferview_dealloc}, + {Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf}, + // We don't bother with Py_bf_releasebuffer since we don't need it. + {0, NULL}, +}; + +static PyType_Spec XIBufferViewType_spec = { + .name = MODULE_NAME ".CrossInterpreterBufferView", + .basicsize = sizeof(XIBufferViewObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + .slots = XIBufferViewType_slots, +}; + + +static PyTypeObject * _get_current_xibufferview_type(void); + +static PyObject * +_memoryview_from_xid(_PyCrossInterpreterData *data) +{ + PyTypeObject *cls = _get_current_xibufferview_type(); + if (cls == NULL) { + return NULL; + } + PyObject *obj = xibufferview_from_xid(cls, data); + if (obj == NULL) { + return NULL; + } + return PyMemoryView_FromObject(obj); +} + +static int +_memoryview_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); + if (view == NULL) { + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) { + PyMem_RawFree(view); + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, + _memoryview_from_xid); + return 0; +} + +static int +register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) +{ + // XIBufferView + assert(*p_state == NULL); + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &XIBufferViewType_spec, NULL); + if (cls == NULL) { + return -1; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return -1; + } + *p_state = cls; + + // Register XID for the builtin memoryview type. + if (_PyCrossInterpreterData_RegisterClass( + &PyMemoryView_Type, _memoryview_shared) < 0) { + return -1; + } + // We don't ever bother un-registering memoryview. + + return 0; +} + + /* module state *************************************************************/ typedef struct { int _notused; + + /* heap types */ + PyTypeObject *XIBufferViewType; } module_state; static inline module_state * @@ -137,19 +296,51 @@ get_module_state(PyObject *mod) return state; } +static module_state * +_get_current_module_state(void) +{ + PyObject *mod = _get_current_module(); + if (mod == NULL) { + // XXX import it? + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME " module not imported yet"); + return NULL; + } + module_state *state = get_module_state(mod); + Py_DECREF(mod); + return state; +} + static int traverse_module_state(module_state *state, visitproc visit, void *arg) { + /* heap types */ + Py_VISIT(state->XIBufferViewType); + return 0; } static int clear_module_state(module_state *state) { + /* heap types */ + Py_CLEAR(state->XIBufferViewType); + return 0; } +static PyTypeObject * +_get_current_xibufferview_type(void) +{ + module_state *state = _get_current_module_state(); + if (state == NULL) { + return NULL; + } + return state->XIBufferViewType; +} + + /* Python code **************************************************************/ static const char * @@ -863,6 +1054,8 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { + module_state *state = get_module_state(mod); + // exceptions if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { goto error; @@ -871,6 +1064,10 @@ module_exec(PyObject *mod) goto error; } + if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) { + goto error; + } + return 0; error: From cc20f1f10c65d5a28930c8a283e6a28a385ee5e1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 15:27:06 -0700 Subject: [PATCH 13/26] Update CODEOWNERS. --- .github/CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1925363cbeb46e..84eceb5089c819 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -78,6 +78,11 @@ Python/traceback.c @iritkatriel **/*importlib/resources/* @jaraco @warsaw @FFY00 **/importlib/metadata/* @jaraco @warsaw +# Subinterpreters +Lib/test/support/interpreters/** @ericsnowcurrently +Modules/_xx*interp*module.c @ericsnowcurrently +Lib/test/test_interpreters/** @ericsnowcurrently + # Dates and times **/*datetime* @pganssle @abalkin **/*str*time* @pganssle @abalkin From cb0a6059c980a5cef413935e12d3b16fbd9e8543 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 15:32:38 -0700 Subject: [PATCH 14/26] Ignore static builtin exception types. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index aa8ce49ae86376..e3a1b5d532bda2 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -290,6 +290,10 @@ Objects/exceptions.c - PyExc_UnicodeWarning - Objects/exceptions.c - PyExc_BytesWarning - Objects/exceptions.c - PyExc_ResourceWarning - Objects/exceptions.c - PyExc_EncodingWarning - +Python/crossinterp.c - _PyExc_InterpreterError - +Python/crossinterp.c - _PyExc_InterpreterNotFoundError - +Python/crossinterp.c - PyExc_InterpreterError - +Python/crossinterp.c - PyExc_InterpreterNotFoundError - ##----------------------- ## singletons From 62a9bacb5051f8c3318907182ced1b6a459d4449 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 15:45:25 -0700 Subject: [PATCH 15/26] Make CODEOWNERS more specific. --- .github/CODEOWNERS | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 84eceb5089c819..f8ebeb4181da00 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -153,7 +153,15 @@ Doc/c-api/stable.rst @encukou **/*itertools* @rhettinger **/*collections* @rhettinger **/*random* @rhettinger -**/*queue* @rhettinger +Doc/**/*queue* @rhettinger +PCbuild/**/*queue* @rhettinger +Modules/_queuemodule.c @rhettinger +Lib/*queue*.py @rhettinger +Lib/asyncio/*queue*.py @rhettinger +Lib/multiprocessing/*queue*.py @rhettinger +Lib/test/*queue*.py @rhettinger +Lib/test_asyncio/*queue*.py @rhettinger +Lib/test_multiprocessing/*queue*.py @rhettinger **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger From 6c7101856587976884d1455976ccc0efeb2ca9cc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 16:05:59 -0700 Subject: [PATCH 16/26] Fix submodule names. --- Lib/test/support/interpreters/__init__.py | 2 +- Lib/test/support/interpreters/{channel.py => channels.py} | 0 Lib/test/support/interpreters/{queue.py => queues.py} | 0 .../test_interpreters/{test_channel.py => test_channels.py} | 6 +++--- .../test_interpreters/{test_queue.py => test_queues.py} | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) rename Lib/test/support/interpreters/{channel.py => channels.py} (100%) rename Lib/test/support/interpreters/{queue.py => queues.py} (100%) rename Lib/test/test_interpreters/{test_channel.py => test_channels.py} (98%) rename Lib/test/test_interpreters/{test_queue.py => test_queues.py} (98%) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index f7a7b7053eeec6..e54982e16439e5 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -8,7 +8,7 @@ InterpreterError, InterpreterNotFoundError, is_shareable, ) -from .queue import ( +from .queues import ( create as create_queue, Queue, QueueEmpty, QueueFull, ) diff --git a/Lib/test/support/interpreters/channel.py b/Lib/test/support/interpreters/channels.py similarity index 100% rename from Lib/test/support/interpreters/channel.py rename to Lib/test/support/interpreters/channels.py diff --git a/Lib/test/support/interpreters/queue.py b/Lib/test/support/interpreters/queues.py similarity index 100% rename from Lib/test/support/interpreters/queue.py rename to Lib/test/support/interpreters/queues.py diff --git a/Lib/test/test_interpreters/test_channel.py b/Lib/test/test_interpreters/test_channels.py similarity index 98% rename from Lib/test/test_interpreters/test_channel.py rename to Lib/test/test_interpreters/test_channels.py index f38237e0b153b9..3c3e18832d4168 100644 --- a/Lib/test/test_interpreters/test_channel.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -7,7 +7,7 @@ # Raise SkipTest if subinterpreters not supported. _channels = import_helper.import_module('_xxinterpchannels') from test.support import interpreters -import test.support.interpreters.channel as channels +from test.support.interpreters import channels from .utils import _run_output, TestBase @@ -121,7 +121,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() interp.exec_sync(dedent(""" - import test.support.interpreters.channel as channels + from test.support.interpreters import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -194,7 +194,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() interp.exec_sync(dedent(""" - import test.support.interpreters.channel as channels + from test.support.interpreters import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) diff --git a/Lib/test/test_interpreters/test_queue.py b/Lib/test/test_interpreters/test_queues.py similarity index 98% rename from Lib/test/test_interpreters/test_queue.py rename to Lib/test/test_interpreters/test_queues.py index cf6ac5f0990d2f..2af90b14d3e3c4 100644 --- a/Lib/test/test_interpreters/test_queue.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -8,7 +8,7 @@ import_helper.import_module('_xxinterpchannels') #import_helper.import_module('_xxinterpqueues') from test.support import interpreters -import test.support.interpreters.queue as queues +from test.support.interpreters import queues from .utils import _run_output, TestBase @@ -170,7 +170,7 @@ def test_get_nowait(self): def test_put_get_same_interpreter(self): interp = interpreters.create() interp.exec_sync(dedent(""" - import test.support.interpreters.queue as queues + from test.support.interpreters import queues queue = queues.create() orig = b'spam' queue.put(orig) From 13a44e353bb9cbee1f5bb26d4ea74c7200db39fc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2023 13:45:09 -0700 Subject: [PATCH 17/26] Use interpreters.__getattr__() for submodule aliases. --- Lib/test/support/interpreters/__init__.py | 21 ++++++++++++++++----- Lib/test/test_interpreters/test_api.py | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index e54982e16439e5..f2a86023a83be6 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -8,20 +8,31 @@ InterpreterError, InterpreterNotFoundError, is_shareable, ) -from .queues import ( - create as create_queue, - Queue, QueueEmpty, QueueFull, -) __all__ = [ 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', 'Interpreter', 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', - 'create_queue', 'QueueEmpty', 'QueueFull', + 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', ] +_queuemod = None + +def __getattr__(name): + if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): + global create_queue, Queue, QueueEmpty, QueueFull + ns = globals() + from .queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, + ) + return ns[name] + else: + raise AttributeError(name) + + class ExecFailure(RuntimeError): def __init__(self, excinfo): diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index d94a03bccf2e97..53e5db3e30a8e1 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -12,6 +12,24 @@ from .utils import _captured_script, _run_output, _running, TestBase +class ModuleTests(TestBase): + + def test_queue_aliases(self): + first = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + second = [ + interpreters.create_queue, + interpreters.Queue, + interpreters.QueueEmpty, + interpreters.QueueFull, + ] + self.assertEqual(second, first) + + class CreateTests(TestBase): def test_in_main(self): From c9d15d2c56f4a688269ee51da34f4d9e30b5e444 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 16:18:03 -0700 Subject: [PATCH 18/26] Fix InterpreterIDTests. --- Lib/test/test_capi/test_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index dec68977a6ba45..3d8c7886a7cb2e 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2079,7 +2079,7 @@ def test_does_not_exist(self): with self.assertRaises(_interpreters.InterpreterNotFoundError): self.InterpreterID(int(id) + 1) # unforced - def test_does_not_exist(self): + def test_destroyed(self): id = _interpreters.create() _interpreters.destroy(id) with self.assertRaises(_interpreters.InterpreterNotFoundError): From 1c76c4a702be612208a94e5849c6d5b4661a07fe Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 16:20:16 -0700 Subject: [PATCH 19/26] LONG_LONG_MAX -> LLONG_MAX --- Modules/_xxsubinterpretersmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 8205fa78820581..37959e953ee4f5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -54,7 +54,7 @@ pylong_to_interpid(PyObject *idobj) // For now, we don't worry about if LLONG_MAX < INT64_MAX. goto bad_id; } -#if LONG_LONG_MAX > INT64_MAX +#if LLONG_MAX > INT64_MAX if (id > INT64_MAX) { goto bad_id; } @@ -105,7 +105,7 @@ look_up_interp(PyObject *arg) static PyObject * interpid_to_pylong(int64_t id) { - assert(id < LONG_LONG_MAX); + assert(id < LLONG_MAX); return PyLong_FromLongLong(id); } From cfae15f4e3742b7eaf39e4dce2d7f5c843dbf2eb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 16:50:04 -0700 Subject: [PATCH 20/26] Fix Interpreter.run(). --- Lib/test/support/interpreters/__init__.py | 1 + Lib/test/test_interpreters/test_api.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index f2a86023a83be6..2d6376deb5907e 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -1,5 +1,6 @@ """Subinterpreters High Level Module.""" +import threading import weakref import _xxsubinterpreters as _interpreters diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 53e5db3e30a8e1..7f4014c28c70ef 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -557,6 +557,24 @@ def task(): # Interpreter.exec_sync() behavior. +class TestInterpreterRun(TestBase): + + def test_success(self): + interp = interpreters.create() + script, file = _captured_script('print("it worked!", end="")') + with file: + t = interp.run(script) + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_failure(self): + interp = interpreters.create() + t = interp.run('raise Exception') + t.join() + + class TestIsShareable(TestBase): def test_default_shareables(self): From 3f98ce1e572c36b972dd0f4b425ca8bfa85a04a9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 16:54:54 -0700 Subject: [PATCH 21/26] Use exec_sync() in test_sys. --- Lib/test/test_sys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 7f49fb004272bb..3524aebca3deec 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -715,7 +715,7 @@ def test_subinterp_intern_dynamically_allocated(self): self.assertIs(t, s) interp = interpreters.create() - interp.run(textwrap.dedent(f''' + interp.exec_sync(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) != {id(s)}, (id(t), {id(s)}) @@ -730,7 +730,7 @@ def test_subinterp_intern_statically_allocated(self): t = sys.intern(s) interp = interpreters.create() - interp.run(textwrap.dedent(f''' + interp.exec_sync(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) == {id(t)}, (id(t), {id(t)}) From f8b1685b2c021057ab9c1feb12c23ebf7f1b28de Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 16:59:07 -0700 Subject: [PATCH 22/26] Fix test_capi. --- Lib/test/test_capi/test_misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3d8c7886a7cb2e..9fea5376687d08 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1527,6 +1527,7 @@ def test_isolated_subinterpreter(self): maxtext = 250 main_interpid = 0 interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) _interpreters.run_string(interpid, f"""if True: import json import os From 0b34a83fa490c4859b9037a4156b4afb29719aee Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 17:05:00 -0700 Subject: [PATCH 23/26] Fix test_importlib. --- Lib/test/test_importlib/test_util.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 914176559806f4..fe5e7b31d9c32b 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -657,14 +657,26 @@ class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): interpid = _interpreters.create(isolated=True) - excsnap = _interpreters.run_string(interpid, script) + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) if excsnap is not None: if excsnap.type.__name__ == 'ImportError': raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): interpid = _interpreters.create(isolated=False) - excsnap = _interpreters.run_string(interpid, script) + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) if excsnap is not None: if excsnap.type.__name__ == 'ImportError': raise ImportError(excsnap.msg) From 68faf1aabb40adcefbf5f9b7840e05dc52049bd4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 17:08:15 -0700 Subject: [PATCH 24/26] Fix test_import. --- Lib/test/test_import/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 1ecac4f37fe1c1..d9dd897f928fc7 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1968,6 +1968,7 @@ def test_disallowed_reimport(self): print(_testsinglephase) ''') interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) excsnap = _interpreters.run_string(interpid, script) self.assertIsNot(excsnap, None) @@ -2102,12 +2103,18 @@ def re_load(self, name, mod): def add_subinterpreter(self): interpid = _interpreters.create(isolated=False) - _interpreters.run_string(interpid, textwrap.dedent(''' + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + _interpreters.exec(interpid, textwrap.dedent(''' import sys import _testinternalcapi ''')) def clean_up(): - _interpreters.run_string(interpid, textwrap.dedent(f''' + _interpreters.exec(interpid, textwrap.dedent(f''' name = {self.NAME!r} if name in sys.modules: sys.modules.pop(name)._clear_globals() From d161f760ae9df5563dee6db7ad85a6c687ef9290 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 17:11:50 -0700 Subject: [PATCH 25/26] Fix test_threading. --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 146e2dbc0fc396..a5744a4037ecea 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1365,7 +1365,7 @@ def test_threads_join_with_no_main(self): DONE = b'D' interp = interpreters.create() - interp.run(f"""if True: + interp.exec_sync(f"""if True: import os import threading import time From 778276f962990cdadd828dc4eda5d255fd087fe2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2023 18:42:07 -0700 Subject: [PATCH 26/26] Fix TestInterpreterRun. --- Lib/test/test_interpreters/test_api.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 7f4014c28c70ef..e4ae9d005b5282 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -570,9 +570,19 @@ def test_success(self): self.assertEqual(out, 'it worked!') def test_failure(self): - interp = interpreters.create() - t = interp.run('raise Exception') - t.join() + caught = False + def excepthook(args): + nonlocal caught + caught = True + threading.excepthook = excepthook + try: + interp = interpreters.create() + t = interp.run('raise Exception') + t.join() + + self.assertTrue(caught) + except BaseException: + threading.excepthook = threading.__excepthook__ class TestIsShareable(TestBase): pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy