diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst
new file mode 100644
index 00000000000000..bbc945016e2880
--- /dev/null
+++ b/Doc/library/_interpreters.rst
@@ -0,0 +1,90 @@
+:mod:`_interpreters` --- Low-level interpreters API
+===================================================
+
+.. module:: _interpreters
+ :synopsis: Low-level interpreters API.
+
+.. versionadded:: 3,7
+
+--------------
+
+This module provides low-level primitives for working with multiple
+Python interpreters in the same runtime in the current process.
+
+More information about (sub)interpreters is found at
+:ref:`sub-interpreter-support`, including what data is shared between
+interpreters and what is unique. Note particularly that interpreters
+aren't inherently threaded, even though they track and manage Python
+threads. To run code in an interpreter in a different OS thread, call
+:func:`run_string` in a function that you run in a new Python thread.
+For example::
+
+ id = _interpreters.create()
+ def f():
+ _interpreters.run_string(id, 'print("in a thread")')
+
+ t = threading.Thread(target=f)
+ t.start()
+
+This module is optional. It is provided by Python implementations which
+support multiple interpreters.
+
+It defines the following functions:
+
+.. function:: enumerate()
+
+ Return a list of the IDs of every existing interpreter.
+
+
+.. function:: get_current()
+
+ Return the ID of the currently running interpreter.
+
+
+.. function:: get_main()
+
+ Return the ID of the main interpreter.
+
+
+.. function:: is_running(id)
+
+ Return whether or not the identified interpreter is currently
+ running any code.
+
+
+.. function:: create()
+
+ Initialize a new Python interpreter and return its identifier. The
+ interpreter will be created in the current thread and will remain
+ idle until something is run in it.
+
+
+.. function:: destroy(id)
+
+ Finalize and destroy the identified interpreter.
+
+
+.. function:: run_string(id, command)
+
+ A wrapper around :c:func:`PyRun_SimpleString` which runs the provided
+ Python program in the main thread of the identified interpreter.
+ Providing an invalid or unknown ID results in a RuntimeError,
+ likewise if the main interpreter or any other running interpreter
+ is used.
+
+ Any value returned from the code is thrown away, similar to what
+ threads do. If the code results in an exception then that exception
+ is raised in the thread in which run_string() was called, similar to
+ how :func:`exec` works. This aligns with how interpreters are not
+ inherently threaded. Note that SystemExit (as raised by sys.exit())
+ is not treated any differently and will result in the process ending
+ if not caught explicitly.
+
+
+.. function:: run_string_unrestricted(id, command, ns=None)
+
+ Like :c:func:`run_string` but returns the dict in which the code
+ was executed. It also supports providing a namespace that gets
+ merged into the execution namespace before execution. Note that
+ this allows objects to leak between interpreters, which may not
+ be desirable.
diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst
index 826bf86d081793..fafbf92c6b0181 100644
--- a/Doc/library/concurrency.rst
+++ b/Doc/library/concurrency.rst
@@ -29,3 +29,4 @@ The following are support modules for some of the above services:
_thread.rst
_dummy_thread.rst
dummy_threading.rst
+ _interpreters.rst
diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h
index b93342120477f3..7816665bb00df5 100644
--- a/Include/internal/pystate.h
+++ b/Include/internal/pystate.h
@@ -80,6 +80,20 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate(
PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config);
+/* Cross-interpreter data sharing */
+
+struct _cid;
+
+typedef struct _cid {
+ void *data;
+ PyObject *(*new_object)(struct _cid *);
+ void (*free)(void *);
+
+ PyInterpreterState *interp;
+ PyObject *object;
+} _PyCrossInterpreterData;
+
+
/* Full Python runtime state */
typedef struct pyruntimestate {
diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py
new file mode 100644
index 00000000000000..29d04aa4d00cfb
--- /dev/null
+++ b/Lib/test/test__interpreters.py
@@ -0,0 +1,582 @@
+import contextlib
+import os
+import os.path
+import shutil
+import tempfile
+from textwrap import dedent, indent
+import threading
+import unittest
+
+from test import support
+from test.support import script_helper
+
+interpreters = support.import_module('_interpreters')
+
+
+SCRIPT_THREADED_INTERP = """\
+from textwrap import dedent
+import threading
+import _interpreters
+def f():
+ _interpreters.run_string(id, dedent('''
+ {}
+ '''))
+
+t = threading.Thread(target=f)
+t.start()
+"""
+
+
+@contextlib.contextmanager
+def _blocked(dirname):
+ filename = os.path.join(dirname, '.lock')
+ wait_script = dedent("""
+ import os.path
+ import time
+ while not os.path.exists('{}'):
+ time.sleep(0.1)
+ """).format(filename)
+ try:
+ yield wait_script
+ finally:
+ support.create_empty_file(filename)
+
+
+class InterpreterTests(unittest.TestCase):
+
+ def setUp(self):
+ self.dirname = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.dirname)
+
+ def test_still_running_at_exit(self):
+ subscript = dedent("""
+ import time
+ # Give plenty of time for the main interpreter to finish.
+ time.sleep(1_000_000)
+ """)
+ script = SCRIPT_THREADED_INTERP.format(indent(subscript, ' '))
+ filename = script_helper.make_script(self.dirname, 'interp', script)
+ with script_helper.spawn_python(filename) as proc:
+ retcode = proc.wait()
+
+ self.assertEqual(retcode, 0)
+
+
+class TestBase(unittest.TestCase):
+
+ def tearDown(self):
+ for id in interpreters.enumerate():
+ if id == 0: # main
+ continue
+ try:
+ interpreters.destroy(id)
+ except RuntimeError:
+ pass # already destroyed
+
+
+class EnumerateTests(TestBase):
+
+ def test_multiple(self):
+ main, = interpreters.enumerate()
+ id1 = interpreters.create()
+ id2 = interpreters.create()
+ ids = interpreters.enumerate()
+
+ self.assertEqual(set(ids), {main, id1, id2})
+
+ def test_main_only(self):
+ main, = interpreters.enumerate()
+
+ self.assertEqual(main, 0)
+
+
+class GetCurrentTests(TestBase):
+
+ def test_main(self):
+ main, = interpreters.enumerate()
+ id = interpreters.get_current()
+
+ self.assertEqual(id, main)
+
+ def test_sub(self):
+ id1 = interpreters.create()
+ ns = interpreters.run_string_unrestricted(id1, dedent("""
+ import _interpreters
+ id = _interpreters.get_current()
+ """))
+ id2 = ns['id']
+
+ self.assertEqual(id2, id1)
+
+
+class GetMainTests(TestBase):
+
+ def test_main(self):
+ expected, = interpreters.enumerate()
+ main = interpreters.get_main()
+
+ self.assertEqual(main, 0)
+ self.assertEqual(main, expected)
+
+
+class IsRunningTests(TestBase):
+
+ def test_main_running(self):
+ main, = interpreters.enumerate()
+ sub = interpreters.create()
+ main_running = interpreters.is_running(main)
+ sub_running = interpreters.is_running(sub)
+
+ self.assertTrue(main_running)
+ self.assertFalse(sub_running)
+
+ def test_sub_running(self):
+ main, = interpreters.enumerate()
+ sub1 = interpreters.create()
+ sub2 = interpreters.create()
+ ns = interpreters.run_string_unrestricted(sub1, dedent(f"""
+ import _interpreters
+ main = _interpreters.is_running({main})
+ sub1 = _interpreters.is_running({sub1})
+ sub2 = _interpreters.is_running({sub2})
+ """))
+ main_running = ns['main']
+ sub1_running = ns['sub1']
+ sub2_running = ns['sub2']
+
+ self.assertTrue(main_running)
+ self.assertTrue(sub1_running)
+ self.assertFalse(sub2_running)
+
+
+class CreateTests(TestBase):
+
+ def test_in_main(self):
+ id = interpreters.create()
+
+ self.assertIn(id, interpreters.enumerate())
+
+ @unittest.skip('enable this test when working on pystate.c')
+ def test_unique_id(self):
+ seen = set()
+ for _ in range(100):
+ id = interpreters.create()
+ interpreters.destroy(id)
+ seen.add(id)
+
+ self.assertEqual(len(seen), 100)
+
+ def test_in_thread(self):
+ lock = threading.Lock()
+ id = None
+ def f():
+ nonlocal id
+ id = interpreters.create()
+ lock.acquire()
+ lock.release()
+
+ t = threading.Thread(target=f)
+ with lock:
+ t.start()
+ t.join()
+ self.assertIn(id, interpreters.enumerate())
+
+ def test_in_subinterpreter(self):
+ main, = interpreters.enumerate()
+ id1 = interpreters.create()
+ ns = interpreters.run_string_unrestricted(id1, dedent("""
+ import _interpreters
+ id = _interpreters.create()
+ """))
+ id2 = ns['id']
+
+ self.assertEqual(set(interpreters.enumerate()), {main, id1, id2})
+
+ def test_in_threaded_subinterpreter(self):
+ main, = interpreters.enumerate()
+ id1 = interpreters.create()
+ ns = None
+ script = dedent("""
+ import _interpreters
+ id = _interpreters.create()
+ """)
+ def f():
+ nonlocal ns
+ ns = interpreters.run_string_unrestricted(id1, script)
+
+ t = threading.Thread(target=f)
+ t.start()
+ t.join()
+ id2 = ns['id']
+
+ self.assertEqual(set(interpreters.enumerate()), {main, id1, id2})
+
+
+ def test_after_destroy_all(self):
+ before = set(interpreters.enumerate())
+ # Create 3 subinterpreters.
+ ids = []
+ for _ in range(3):
+ id = interpreters.create()
+ ids.append(id)
+ # Now destroy them.
+ for id in ids:
+ interpreters.destroy(id)
+ # Finally, create another.
+ id = interpreters.create()
+ self.assertEqual(set(interpreters.enumerate()), before | {id})
+
+ def test_after_destroy_some(self):
+ before = set(interpreters.enumerate())
+ # Create 3 subinterpreters.
+ id1 = interpreters.create()
+ id2 = interpreters.create()
+ id3 = interpreters.create()
+ # Now destroy 2 of them.
+ interpreters.destroy(id1)
+ interpreters.destroy(id3)
+ # Finally, create another.
+ id = interpreters.create()
+ self.assertEqual(set(interpreters.enumerate()), before | {id, id2})
+
+
+class DestroyTests(TestBase):
+
+ def test_one(self):
+ id1 = interpreters.create()
+ id2 = interpreters.create()
+ id3 = interpreters.create()
+ self.assertIn(id2, interpreters.enumerate())
+ interpreters.destroy(id2)
+ self.assertNotIn(id2, interpreters.enumerate())
+ self.assertIn(id1, interpreters.enumerate())
+ self.assertIn(id3, interpreters.enumerate())
+
+ def test_all(self):
+ before = set(interpreters.enumerate())
+ ids = set()
+ for _ in range(3):
+ id = interpreters.create()
+ ids.add(id)
+ self.assertEqual(set(interpreters.enumerate()), before | ids)
+ for id in ids:
+ interpreters.destroy(id)
+ self.assertEqual(set(interpreters.enumerate()), before)
+
+ def test_main(self):
+ main, = interpreters.enumerate()
+ with self.assertRaises(RuntimeError):
+ interpreters.destroy(main)
+
+ def f():
+ with self.assertRaises(RuntimeError):
+ interpreters.destroy(main)
+
+ t = threading.Thread(target=f)
+ t.start()
+ t.join()
+
+ def test_already_destroyed(self):
+ id = interpreters.create()
+ interpreters.destroy(id)
+ with self.assertRaises(RuntimeError):
+ interpreters.destroy(id)
+
+ def test_does_not_exist(self):
+ with self.assertRaises(RuntimeError):
+ interpreters.destroy(1_000_000)
+
+ def test_bad_id(self):
+ with self.assertRaises(RuntimeError):
+ interpreters.destroy(-1)
+
+ def test_from_current(self):
+ main, = interpreters.enumerate()
+ id = interpreters.create()
+ script = dedent("""
+ import _interpreters
+ _interpreters.destroy({})
+ """).format(id)
+
+ with self.assertRaises(RuntimeError):
+ interpreters.run_string(id, script)
+ self.assertEqual(set(interpreters.enumerate()), {main, id})
+
+ def test_from_sibling(self):
+ main, = interpreters.enumerate()
+ id1 = interpreters.create()
+ id2 = interpreters.create()
+ script = dedent("""
+ import _interpreters
+ _interpreters.destroy({})
+ """).format(id2)
+ interpreters.run_string(id1, script)
+
+ self.assertEqual(set(interpreters.enumerate()), {main, id1})
+
+ def test_from_other_thread(self):
+ id = interpreters.create()
+ def f():
+ interpreters.destroy(id)
+
+ t = threading.Thread(target=f)
+ t.start()
+ t.join()
+
+ def test_still_running(self):
+ # XXX Rewrite this test without files by using
+ # run_string_unrestricted().
+ main, = interpreters.enumerate()
+ id = interpreters.create()
+ def f():
+ interpreters.run_string(id, wait_script)
+
+ dirname = tempfile.mkdtemp()
+ t = threading.Thread(target=f)
+ with _blocked(dirname) as wait_script:
+ t.start()
+ with self.assertRaises(RuntimeError):
+ interpreters.destroy(id)
+
+ t.join()
+ self.assertEqual(set(interpreters.enumerate()), {main, id})
+
+
+class RunStringTests(TestBase):
+
+ SCRIPT = dedent("""
+ with open('{}', 'w') as out:
+ out.write('{}')
+ """)
+ FILENAME = 'spam'
+
+ def setUp(self):
+ self.id = interpreters.create()
+ self.dirname = None
+ self.filename = None
+
+ def tearDown(self):
+ if self.dirname is not None:
+ try:
+ shutil.rmtree(self.dirname)
+ except FileNotFoundError:
+ pass # already deleted
+ super().tearDown()
+
+ def _resolve_filename(self, name=None):
+ if name is None:
+ name = self.FILENAME
+ if self.dirname is None:
+ self.dirname = tempfile.mkdtemp()
+ return os.path.join(self.dirname, name)
+
+ def _empty_file(self):
+ self.filename = self._resolve_filename()
+ support.create_empty_file(self.filename)
+ return self.filename
+
+ def assert_file_contains(self, expected, filename=None):
+ if filename is None:
+ filename = self.filename
+ self.assertIsNotNone(filename)
+ with open(filename) as out:
+ content = out.read()
+ self.assertEqual(content, expected)
+
+ def test_success(self):
+ filename = self._empty_file()
+ expected = 'spam spam spam spam spam'
+ script = self.SCRIPT.format(filename, expected)
+ interpreters.run_string(self.id, script)
+
+ self.assert_file_contains(expected)
+
+ def test_in_thread(self):
+ filename = self._empty_file()
+ expected = 'spam spam spam spam spam'
+ script = self.SCRIPT.format(filename, expected)
+ def f():
+ interpreters.run_string(self.id, script)
+
+ t = threading.Thread(target=f)
+ t.start()
+ t.join()
+
+ self.assert_file_contains(expected)
+
+ def test_create_thread(self):
+ filename = self._empty_file()
+ expected = 'spam spam spam spam spam'
+ script = dedent("""
+ import threading
+ def f():
+ with open('{}', 'w') as out:
+ out.write('{}')
+
+ t = threading.Thread(target=f)
+ t.start()
+ t.join()
+ """).format(filename, expected)
+ interpreters.run_string(self.id, script)
+
+ self.assert_file_contains(expected)
+
+ @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()")
+ def test_fork(self):
+ filename = self._empty_file()
+ expected = 'spam spam spam spam spam'
+ script = dedent("""
+ import os
+ r, w = os.pipe()
+ pid = os.fork()
+ if pid == 0: # child
+ import sys
+ filename = '{}'
+ with open(filename, 'w') as out:
+ out.write('{}')
+ os.write(w, b'done!')
+
+ # Kill the unittest runner in the child process.
+ os._exit(1)
+ else:
+ import select
+ try:
+ select.select([r], [], [])
+ finally:
+ os.close(r)
+ os.close(w)
+ """).format(filename, expected)
+ interpreters.run_string(self.id, script)
+ self.assert_file_contains(expected)
+
+ def test_already_running(self):
+ def f():
+ interpreters.run_string(self.id, wait_script)
+
+ t = threading.Thread(target=f)
+ dirname = tempfile.mkdtemp()
+ with _blocked(dirname) as wait_script:
+ t.start()
+ with self.assertRaises(RuntimeError):
+ interpreters.run_string(self.id, 'print("spam")')
+ t.join()
+
+ def test_does_not_exist(self):
+ id = 0
+ while id in interpreters.enumerate():
+ id += 1
+ with self.assertRaises(RuntimeError):
+ interpreters.run_string(id, 'print("spam")')
+
+ def test_error_id(self):
+ with self.assertRaises(RuntimeError):
+ interpreters.run_string(-1, 'print("spam")')
+
+ def test_bad_id(self):
+ with self.assertRaises(TypeError):
+ interpreters.run_string('spam', 'print("spam")')
+
+ def test_bad_code(self):
+ with self.assertRaises(TypeError):
+ interpreters.run_string(self.id, 10)
+
+ def test_bytes_for_code(self):
+ with self.assertRaises(TypeError):
+ interpreters.run_string(self.id, b'print("spam")')
+
+ def test_invalid_syntax(self):
+ with self.assertRaises(SyntaxError):
+ # missing close paren
+ interpreters.run_string(self.id, 'print("spam"')
+
+ def test_failure(self):
+ with self.assertRaises(Exception) as caught:
+ interpreters.run_string(self.id, 'raise Exception("spam")')
+ self.assertEqual(str(caught.exception), 'spam')
+
+ def test_sys_exit(self):
+ with self.assertRaises(SystemExit) as cm:
+ interpreters.run_string(self.id, dedent("""
+ import sys
+ sys.exit()
+ """))
+ self.assertIsNone(cm.exception.code)
+
+ with self.assertRaises(SystemExit) as cm:
+ interpreters.run_string(self.id, dedent("""
+ import sys
+ sys.exit(42)
+ """))
+ self.assertEqual(cm.exception.code, 42)
+
+ def test_SystemError(self):
+ with self.assertRaises(SystemExit) as cm:
+ interpreters.run_string(self.id, 'raise SystemExit(42)')
+ self.assertEqual(cm.exception.code, 42)
+
+
+class RunStringUnrestrictedTests(TestBase):
+
+ def setUp(self):
+ self.id = interpreters.create()
+
+ def test_without_ns(self):
+ script = dedent("""
+ spam = 42
+ """)
+ ns = interpreters.run_string_unrestricted(self.id, script)
+
+ self.assertEqual(ns['spam'], 42)
+
+ def test_with_ns(self):
+ updates = {'spam': 'ham', 'eggs': -1}
+ script = dedent("""
+ spam = 42
+ result = spam + eggs
+ """)
+ ns = interpreters.run_string_unrestricted(self.id, script, updates)
+
+ self.assertEqual(ns['spam'], 42)
+ self.assertEqual(ns['eggs'], -1)
+ self.assertEqual(ns['result'], 41)
+
+ def test_ns_does_not_overwrite(self):
+ updates = {'__name__': 'not __main__'}
+ script = dedent("""
+ spam = 42
+ """)
+ ns = interpreters.run_string_unrestricted(self.id, script, updates)
+
+ self.assertEqual(ns['__name__'], '__main__')
+
+ def test_main_not_shared(self):
+ ns1 = interpreters.run_string_unrestricted(self.id, 'spam = True')
+ ns2 = interpreters.run_string_unrestricted(self.id, 'eggs = False')
+
+ self.assertIn('spam', ns1)
+ self.assertNotIn('eggs', ns1)
+ self.assertIn('eggs', ns2)
+ self.assertNotIn('spam', ns2)
+
+ def test_return_execution_namespace(self):
+ script = dedent("""
+ spam = 42
+ """)
+ ns = interpreters.run_string_unrestricted(self.id, script)
+
+ ns.pop('__builtins__')
+ ns.pop('__loader__')
+ self.assertEqual(ns, {
+ '__name__': '__main__',
+ '__annotations__': {},
+ '__doc__': None,
+ '__package__': None,
+ '__spec__': None,
+ 'spam': 42,
+ })
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c
new file mode 100644
index 00000000000000..f9b800cb1ace35
--- /dev/null
+++ b/Modules/_interpretersmodule.c
@@ -0,0 +1,632 @@
+
+/* interpreters module */
+/* low-level access to interpreter primitives */
+
+#include "Python.h"
+#include "frameobject.h"
+#include "internal/pystate.h"
+
+
+static PyInterpreterState *
+_get_current(void)
+{
+ PyThreadState *tstate;
+
+ tstate = PyThreadState_Get();
+ if (tstate == NULL)
+ return NULL;
+ return tstate->interp;
+}
+
+/* sharing-specific functions and structs */
+
+static int
+_PyObject_CheckShareable(PyObject *obj)
+{
+ if (PyBytes_CheckExact(obj))
+ return 0;
+ PyErr_SetString(PyExc_ValueError,
+ "obj is not a cross-interpreter shareable type");
+ return 1;
+}
+
+static PyObject *
+_new_bytes_object(_PyCrossInterpreterData *data)
+{
+ return PyBytes_FromString((char *)(data->data));
+}
+
+static int
+_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data)
+{
+ data->data = (void *)(PyBytes_AS_STRING(obj));
+ data->new_object = _new_bytes_object;
+ data->free = NULL;
+ return 0;
+}
+
+static int
+_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
+{
+ Py_INCREF(obj);
+
+ if (_PyObject_CheckShareable(obj) != 0) {
+ Py_DECREF(obj);
+ return 1;
+ }
+
+ data->interp = _get_current();
+ data->object = obj;
+
+ if (PyBytes_CheckExact(obj)) {
+ if (_bytes_shared(obj, data) != 0) {
+ Py_DECREF(obj);
+ return 1;
+ }
+ }
+
+ return 0;
+};
+
+static void
+_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
+{
+ PyThreadState *save_tstate = NULL;
+ if (data->interp != NULL) {
+ // Switch to the original interpreter.
+ PyThreadState *tstate = PyInterpreterState_ThreadHead(data->interp);
+ save_tstate = PyThreadState_Swap(tstate);
+ }
+
+ if (data->free != NULL) {
+ data->free(data->data);
+ }
+ Py_XDECREF(data->object);
+
+ // Switch back.
+ if (save_tstate != NULL)
+ PyThreadState_Swap(save_tstate);
+}
+
+static PyObject *
+_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
+{
+ return data->new_object(data);
+}
+
+struct _shareditem {
+ Py_UNICODE *name;
+ Py_ssize_t namelen;
+ _PyCrossInterpreterData data;
+};
+
+void
+_sharedns_clear(struct _shareditem *shared)
+{
+ for (struct _shareditem *item=shared; item->name != NULL; item += 1) {
+ _PyCrossInterpreterData_Release(&item->data);
+ }
+}
+
+static struct _shareditem *
+_get_shared_ns(PyObject *shareable)
+{
+ if (shareable == NULL || shareable == Py_None)
+ return NULL;
+ Py_ssize_t len = PyDict_Size(shareable);
+ if (len == 0)
+ return NULL;
+
+ struct _shareditem *shared = PyMem_NEW(struct _shareditem, len+1);
+ Py_ssize_t pos = 0;
+ for (Py_ssize_t i=0; i < len; i++) {
+ PyObject *key, *value;
+ if (PyDict_Next(shareable, &pos, &key, &value) == 0) {
+ break;
+ }
+ struct _shareditem *item = shared + i;
+
+ if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0)
+ break;
+ item->name = PyUnicode_AsUnicodeAndSize(key, &item->namelen);
+ if (item->name == NULL) {
+ _PyCrossInterpreterData_Release(&item->data);
+ break;
+ }
+ (item + 1)->name = NULL; // Mark the next one as the last.
+ }
+ if (PyErr_Occurred()) {
+ _sharedns_clear(shared);
+ PyMem_Free(shared);
+ return NULL;
+ }
+ return shared;
+}
+
+static int
+_shareditem_apply(struct _shareditem *item, PyObject *ns)
+{
+ PyObject *name = PyUnicode_FromUnicode(item->name, item->namelen);
+ if (name == NULL) {
+ return 1;
+ }
+ PyObject *value = _PyCrossInterpreterData_NewObject(&item->data);
+ if (value == NULL) {
+ Py_DECREF(name);
+ return 1;
+ }
+ int res = PyDict_SetItem(ns, name, value);
+ Py_DECREF(name);
+ Py_DECREF(value);
+ return res;
+}
+
+// XXX This cannot use PyObject fields.
+
+struct _shared_exception {
+ PyObject *exc;
+ PyObject *value;
+ PyObject *tb;
+};
+
+static struct _shared_exception *
+_get_shared_exception(void)
+{
+ struct _shared_exception *exc = PyMem_NEW(struct _shared_exception, 1);
+ // XXX Fatal if NULL?
+ PyErr_Fetch(&exc->exc, &exc->value, &exc->tb);
+ return exc;
+}
+
+static void
+_apply_shared_exception(struct _shared_exception *exc)
+{
+ if (PyErr_Occurred()) {
+ _PyErr_ChainExceptions(exc->exc, exc->value, exc->tb);
+ } else {
+ PyErr_Restore(exc->exc, exc->value, exc->tb);
+ }
+
+}
+
+/* interpreter-specific functions */
+
+static PyInterpreterState *
+_look_up_int64(PY_INT64_T requested_id)
+{
+ if (requested_id < 0)
+ goto error;
+
+ PyInterpreterState *interp = PyInterpreterState_Head();
+ while (interp != NULL) {
+ PY_INT64_T id = PyInterpreterState_GetID(interp);
+ if (id < 0)
+ return NULL;
+ if (requested_id == id)
+ return interp;
+ interp = PyInterpreterState_Next(interp);
+ }
+
+error:
+ PyErr_Format(PyExc_RuntimeError,
+ "unrecognized interpreter ID %lld", requested_id);
+ return NULL;
+}
+
+static PyInterpreterState *
+_look_up(PyObject *requested_id)
+{
+ long long id = PyLong_AsLongLong(requested_id);
+ if (id == -1 && PyErr_Occurred() != NULL)
+ return NULL;
+ assert(id <= INT64_MAX);
+ return _look_up_int64(id);
+}
+
+static PyObject *
+_get_id(PyInterpreterState *interp)
+{
+ PY_INT64_T id = PyInterpreterState_GetID(interp);
+ if (id < 0)
+ return NULL;
+ return PyLong_FromLongLong(id);
+}
+
+static int
+_is_running(PyInterpreterState *interp)
+{
+ PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
+ if (PyThreadState_Next(tstate) != NULL) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "interpreter has more than one thread");
+ return -1;
+ }
+ PyFrameObject *frame = _PyThreadState_GetFrame(tstate);
+ if (frame == NULL) {
+ if (PyErr_Occurred() != NULL)
+ return -1;
+ return 0;
+ }
+ return (int)(frame->f_executing);
+}
+
+static int
+_ensure_not_running(PyInterpreterState *interp)
+{
+ int is_running = _is_running(interp);
+ if (is_running < 0)
+ return -1;
+ if (is_running) {
+ PyErr_Format(PyExc_RuntimeError, "interpreter already running");
+ return -1;
+ }
+ return 0;
+}
+
+static int
+_run_script(PyInterpreterState *interp, const char *codestr,
+ struct _shareditem *shared, struct _shared_exception **exc)
+{
+ PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__");
+ if (main_mod == NULL)
+ return -1;
+ PyObject *ns = PyModule_GetDict(main_mod); // borrowed
+ Py_INCREF(ns);
+ Py_DECREF(main_mod);
+ if (ns == NULL)
+ return -1;
+
+ // Apply the cross-interpreter data.
+ if (shared != NULL) {
+ for (struct _shareditem *item=shared; item->name != NULL; item += 1) {
+ if (_shareditem_apply(shared, ns) != 0) {
+ Py_DECREF(ns);
+ return -1;
+ }
+ }
+ }
+
+ // Run the string (see PyRun_SimpleStringFlags).
+ PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL);
+ Py_DECREF(ns);
+ if (result == NULL) {
+ // Get the exception from the subinterpreter.
+ *exc = _get_shared_exception();
+ // XXX Clear the exception?
+ } else {
+ Py_DECREF(result); // We throw away the result.
+ }
+
+ return 0;
+}
+
+static int
+_run_script_in_interpreter(PyInterpreterState *interp, const char *codestr,
+ PyObject *shareable)
+{
+ // XXX lock?
+ if (_ensure_not_running(interp) < 0)
+ return -1;
+
+ struct _shareditem *shared = _get_shared_ns(shareable);
+ if (shared == NULL && PyErr_Occurred())
+ return -1;
+
+ // Switch to interpreter.
+ PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
+ PyThreadState *save_tstate = PyThreadState_Swap(tstate);
+
+ // Run the script.
+ struct _shared_exception *exc = NULL;
+ if (_run_script(interp, codestr, shared, &exc) != 0) {
+ // XXX What to do if the the result isn't 0?
+ }
+
+ // Switch back.
+ if (save_tstate != NULL)
+ PyThreadState_Swap(save_tstate);
+
+ // Propagate any exception out to the caller.
+ if (exc != NULL) {
+ _apply_shared_exception(exc);
+ }
+
+ if (shared != NULL) {
+ _sharedns_clear(shared);
+ PyMem_Free(shared);
+ }
+
+ return 0;
+}
+
+
+/* module level code ********************************************************/
+
+static PyObject *
+interp_create(PyObject *self, PyObject *args)
+{
+ if (!PyArg_UnpackTuple(args, "create", 0, 0))
+ return NULL;
+
+ // Create and initialize the new interpreter.
+ PyThreadState *tstate, *save_tstate;
+ save_tstate = PyThreadState_Swap(NULL);
+ tstate = Py_NewInterpreter();
+ PyThreadState_Swap(save_tstate);
+ if (tstate == NULL) {
+ /* Since no new thread state was created, there is no exception to
+ propagate; raise a fresh one after swapping in the old thread
+ state. */
+ PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed");
+ return NULL;
+ }
+ return _get_id(tstate->interp);
+}
+
+PyDoc_STRVAR(create_doc,
+"create() -> ID\n\
+\n\
+Create a new interpreter and return a unique generated ID.");
+
+
+static PyObject *
+interp_destroy(PyObject *self, PyObject *args)
+{
+ PyObject *id;
+ if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id))
+ return NULL;
+ if (!PyLong_Check(id)) {
+ PyErr_SetString(PyExc_TypeError, "ID must be an int");
+ return NULL;
+ }
+
+ // Look up the interpreter.
+ PyInterpreterState *interp = _look_up(id);
+ if (interp == NULL)
+ return NULL;
+
+ // Ensure we don't try to destroy the current interpreter.
+ PyInterpreterState *current = _get_current();
+ if (current == NULL)
+ return NULL;
+ if (interp == current) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "cannot destroy the current interpreter");
+ return NULL;
+ }
+
+ // Ensure the interpreter isn't running.
+ /* XXX We *could* support destroying a running interpreter but
+ aren't going to worry about it for now. */
+ if (_ensure_not_running(interp) < 0)
+ return NULL;
+
+ // Destroy the interpreter.
+ //PyInterpreterState_Delete(interp);
+ PyThreadState *tstate, *save_tstate;
+ tstate = PyInterpreterState_ThreadHead(interp);
+ save_tstate = PyThreadState_Swap(tstate);
+ Py_EndInterpreter(tstate);
+ PyThreadState_Swap(save_tstate);
+
+ Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(destroy_doc,
+"destroy(ID)\n\
+\n\
+Destroy the identified interpreter.\n\
+\n\
+Attempting to destroy the current interpreter results in a RuntimeError.\n\
+So does an unrecognized ID.");
+
+
+static PyObject *
+interp_enumerate(PyObject *self)
+{
+ PyObject *ids, *id;
+ PyInterpreterState *interp;
+
+ ids = PyList_New(0);
+ if (ids == NULL)
+ return NULL;
+
+ interp = PyInterpreterState_Head();
+ while (interp != NULL) {
+ id = _get_id(interp);
+ if (id == NULL)
+ return NULL;
+ // insert at front of list
+ if (PyList_Insert(ids, 0, id) < 0)
+ return NULL;
+
+ interp = PyInterpreterState_Next(interp);
+ }
+
+ return ids;
+}
+
+PyDoc_STRVAR(enumerate_doc,
+"enumerate() -> [ID]\n\
+\n\
+Return a list containing the ID of every existing interpreter.");
+
+
+static PyObject *
+interp_get_current(PyObject *self)
+{
+ PyInterpreterState *interp =_get_current();
+ if (interp == NULL)
+ return NULL;
+ return _get_id(interp);
+}
+
+PyDoc_STRVAR(get_current_doc,
+"get_current() -> ID\n\
+\n\
+Return the ID of current interpreter.");
+
+
+static PyObject *
+interp_get_main(PyObject *self)
+{
+ // Currently, 0 is always the main interpreter.
+ return PyLong_FromLongLong(0);
+}
+
+PyDoc_STRVAR(get_main_doc,
+"get_main() -> ID\n\
+\n\
+Return the ID of main interpreter.");
+
+
+static PyObject *
+interp_run_string(PyObject *self, PyObject *args)
+{
+ PyObject *id, *code;
+ PyObject *shared = NULL;
+ if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared))
+ return NULL;
+ if (!PyLong_Check(id)) {
+ PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int");
+ return NULL;
+ }
+ if (!PyUnicode_Check(code)) {
+ PyErr_SetString(PyExc_TypeError,
+ "second arg (code) must be a string");
+ return NULL;
+ }
+
+ // Look up the interpreter.
+ PyInterpreterState *interp = _look_up(id);
+ if (interp == NULL)
+ return NULL;
+
+ // Extract code.
+ Py_ssize_t size;
+ const char *codestr = PyUnicode_AsUTF8AndSize(code, &size);
+ if (codestr == NULL)
+ return NULL;
+ if (strlen(codestr) != (size_t)size) {
+ PyErr_SetString(PyExc_ValueError,
+ "source code string cannot contain null bytes");
+ return NULL;
+ }
+
+ // Run the code in the interpreter.
+ if (_run_script_in_interpreter(interp, codestr, shared) != 0)
+ return NULL;
+ Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(run_string_doc,
+"run_string(ID, sourcetext)\n\
+\n\
+Execute the provided string in the identified interpreter.\n\
+\n\
+See PyRun_SimpleStrings.");
+
+
+static PyObject *
+object_is_shareable(PyObject *self, PyObject *args)
+{
+ PyObject *obj;
+ if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj))
+ return NULL;
+ if (_PyObject_CheckShareable(obj) == 0)
+ Py_RETURN_TRUE;
+ PyErr_Clear();
+ Py_RETURN_FALSE;
+}
+
+PyDoc_STRVAR(is_shareable_doc,
+"is_shareable(obj) -> bool\n\
+\n\
+Return True if the object's data may be shared between interpreters and\n\
+False otherwise.");
+
+
+static PyObject *
+interp_is_running(PyObject *self, PyObject *args)
+{
+ PyObject *id;
+ if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id))
+ return NULL;
+ if (!PyLong_Check(id)) {
+ PyErr_SetString(PyExc_TypeError, "ID must be an int");
+ return NULL;
+ }
+
+ PyInterpreterState *interp = _look_up(id);
+ if (interp == NULL)
+ return NULL;
+ int is_running = _is_running(interp);
+ if (is_running < 0)
+ return NULL;
+ if (is_running)
+ Py_RETURN_TRUE;
+ Py_RETURN_FALSE;
+}
+
+PyDoc_STRVAR(is_running_doc,
+"is_running(id) -> bool\n\
+\n\
+Return whether or not the identified interpreter is running.");
+
+
+static PyMethodDef module_functions[] = {
+ {"create", (PyCFunction)interp_create,
+ METH_VARARGS, create_doc},
+ {"destroy", (PyCFunction)interp_destroy,
+ METH_VARARGS, destroy_doc},
+
+ {"enumerate", (PyCFunction)interp_enumerate,
+ METH_NOARGS, enumerate_doc},
+ {"get_current", (PyCFunction)interp_get_current,
+ METH_NOARGS, get_current_doc},
+ {"get_main", (PyCFunction)interp_get_main,
+ METH_NOARGS, get_main_doc},
+ {"is_running", (PyCFunction)interp_is_running,
+ METH_VARARGS, is_running_doc},
+
+ {"run_string", (PyCFunction)interp_run_string,
+ METH_VARARGS, run_string_doc},
+
+ {"is_shareable", (PyCFunction)object_is_shareable,
+ METH_VARARGS, is_shareable_doc},
+
+ {NULL, NULL} /* sentinel */
+};
+
+
+/* initialization function */
+
+PyDoc_STRVAR(module_doc,
+"This module provides primitive operations to manage Python interpreters.\n\
+The 'interpreters' module provides a more convenient interface.");
+
+static struct PyModuleDef interpretersmodule = {
+ PyModuleDef_HEAD_INIT,
+ "_interpreters",
+ module_doc,
+ -1,
+ module_functions,
+ NULL,
+ NULL,
+ NULL,
+ NULL
+};
+
+
+PyMODINIT_FUNC
+PyInit__interpreters(void)
+{
+ PyObject *module;
+
+ module = PyModule_Create(&interpretersmodule);
+ if (module == NULL)
+ return NULL;
+
+
+ return module;
+}
diff --git a/setup.py b/setup.py
index c22de17f953396..eaf3ba218266c0 100644
--- a/setup.py
+++ b/setup.py
@@ -740,6 +740,10 @@ def detect_modules(self):
['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c'])
)
+ # Python interface to subinterpreter C-API.
+ exts.append(Extension('_interpreters', ['_interpretersmodule.c'],
+ define_macros=[('Py_BUILD_CORE', '')]))
+
#
# Here ends the simple stuff. From here on, modules need certain
# libraries, are platform-specific, or present other surprises.
pFad - Phonifier reborn
Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy