From f5b464a8cb689ec0f91c27bc9b5482617f93c60e Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 10 Sep 2019 11:00:52 +0000 Subject: [PATCH 01/20] Add tests for the interpreters module --- Lib/test/test_interpreters.py | 2119 +++++++++++++++++++++++++++++++++ 1 file changed, 2119 insertions(+) create mode 100644 Lib/test/test_interpreters.py diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py new file mode 100644 index 00000000000000..65c6e77d69d1ee --- /dev/null +++ b/Lib/test/test_interpreters.py @@ -0,0 +1,2119 @@ +import interpreters +from collections import namedtuple +import contextlib +import itertools +import os +import pickle +import sys +from textwrap import dedent +import threading +import time +import unittest + +from test import support +from test.support import script_helper + + +################################## +# helpers + +def powerset(*sets): + return itertools.chain.from_iterable( + combinations(sets, r) + for r in range(len(sets)+1)) + + +def _captured_script(script): + r, w = os.pipe() + indented = script.replace('\n', '\n ') + wrapped = dedent(f""" + import contextlib + with open({w}, 'w') as spipe: + with contextlib.redirect_stdout(spipe): + {indented} + """) + return wrapped, open(r) + + +def _run_output(interp, request, shared=None): + script, rpipe = _captured_script(request) + with rpipe: + interpreters.run_string(interp, script, shared) + return rpipe.read() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interpreters.run_string(interp, dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + +#@contextmanager +#def run_threaded(id, source, **shared): +# def run(): +# run_interp(id, source, **shared) +# t = threading.Thread(target=run) +# t.start() +# yield +# t.join() + + +def run_interp(id, source, **shared): + _run_interp(id, source, shared) + + +def _run_interp(id, source, shared, _mainns={}): + source = dedent(source) + main = interpreters.get_main() + if main == id: + if interpreters.get_current() != main: + raise RuntimeError + # XXX Run a func? + exec(source, _mainns) + else: + interpreters.run_string(id, source, shared) + + +def run_interp_threaded(id, source, **shared): + def run(): + _run(id, source, shared) + t = threading.Thread(target=run) + t.start() + t.join() + + +class Interpreter(namedtuple('Interpreter', 'name id')): + + @classmethod + def from_raw(cls, raw): + if isinstance(raw, cls): + return raw + elif isinstance(raw, str): + return cls(raw) + else: + raise NotImplementedError + + def __new__(cls, name=None, id=None): + main = interpreters.get_main() + if id == main: + if not name: + name = 'main' + elif name != 'main': + raise ValueError( + 'name mismatch (expected "main", got "{}")'.format(name)) + id = main + elif id is not None: + if not name: + name = 'interp' + elif name == 'main': + raise ValueError('name mismatch (unexpected "main")') + if not isinstance(id, interpreters.InterpreterID): + id = interpreters.InterpreterID(id) + elif not name or name == 'main': + name = 'main' + id = main + else: + id = interpreters.create() + self = super().__new__(cls, name, id) + return self + + +# XXX expect_channel_closed() is unnecessary once we improve exc propagation. + +@contextlib.contextmanager +def expect_channel_closed(): + try: + yield + except interpreters.ChannelClosedError: + pass + else: + assert False, 'channel not closed' + + +class ChannelAction(namedtuple('ChannelAction', 'action end interp')): + + def __new__(cls, action, end=None, interp=None): + if not end: + end = 'both' + if not interp: + interp = 'main' + self = super().__new__(cls, action, end, interp) + return self + + def __init__(self, *args, **kwargs): + if self.action == 'use': + if self.end not in ('same', 'opposite', 'send', 'recv'): + raise ValueError(self.end) + elif self.action in ('close', 'force-close'): + if self.end not in ('both', 'same', 'opposite', 'send', 'recv'): + raise ValueError(self.end) + else: + raise ValueError(self.action) + if self.interp not in ('main', 'same', 'other', 'extra'): + raise ValueError(self.interp) + + def resolve_end(self, end): + if self.end == 'same': + return end + elif self.end == 'opposite': + return 'recv' if end == 'send' else 'send' + else: + return self.end + + def resolve_interp(self, interp, other, extra): + if self.interp == 'same': + return interp + elif self.interp == 'other': + if other is None: + raise RuntimeError + return other + elif self.interp == 'extra': + if extra is None: + raise RuntimeError + return extra + elif self.interp == 'main': + if interp.name == 'main': + return interp + elif other and other.name == 'main': + return other + else: + raise RuntimeError + # Per __init__(), there aren't any others. + + +class ChannelState(namedtuple('ChannelState', 'pending closed')): + + def __new__(cls, pending=0, *, closed=False): + self = super().__new__(cls, pending, closed) + return self + + def incr(self): + return type(self)(self.pending + 1, closed=self.closed) + + def decr(self): + return type(self)(self.pending - 1, closed=self.closed) + + def close(self, *, force=True): + if self.closed: + if not force or self.pending == 0: + return self + return type(self)(0 if force else self.pending, closed=True) + + +def run_action(cid, action, end, state, *, hideclosed=True): + if state.closed: + if action == 'use' and end == 'recv' and state.pending: + expectfail = False + else: + expectfail = True + else: + expectfail = False + + try: + result = _run_action(cid, action, end, state) + except interpreters.ChannelClosedError: + if not hideclosed and not expectfail: + raise + result = state.close() + else: + if expectfail: + raise ... # XXX + return result + + +def _run_action(cid, action, end, state): + if action == 'use': + if end == 'send': + interpreters.channel_send(cid, b'spam') + return state.incr() + elif end == 'recv': + if not state.pending: + try: + interpreters.channel_recv(cid) + except interpreters.ChannelEmptyError: + return state + else: + raise Exception('expected ChannelEmptyError') + else: + interpreters.channel_recv(cid) + return state.decr() + else: + raise ValueError(end) + elif action == 'close': + kwargs = {} + if end in ('recv', 'send'): + kwargs[end] = True + interpreters.channel_close(cid, **kwargs) + return state.close() + elif action == 'force-close': + kwargs = { + 'force': True, + } + if end in ('recv', 'send'): + kwargs[end] = True + interpreters.channel_close(cid, **kwargs) + return state.close(force=True) + else: + raise ValueError(action) + + +def clean_up_interpreters(): + for id in interpreters.list_all(): + if id == 0: # main + continue + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + +def clean_up_channels(): + for cid in interpreters.channel_list_all(): + try: + interpreters.channel_destroy(cid) + except interpreters.ChannelNotFoundError: + pass # already destroyed + + +class TestBase(unittest.TestCase): + + def tearDown(self): + clean_up_interpreters() + clean_up_channels() + + +################################## +# misc. tests + +class IsShareableTests(unittest.TestCase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + ] + for obj in shareables: + with self.subTest(obj): + self.assertTrue( + interpreters.is_shareable(obj)) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + True, + False, + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + 100.0, + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +class ShareableTypeTests(unittest.TestCase): + + def setUp(self): + super().setUp() + self.cid = interpreters.channel_create() + + def tearDown(self): + interpreters.channel_destroy(self.cid) + super().tearDown() + + def _assert_values(self, values): + for obj in values: + with self.subTest(obj): + interpreters.channel_send(self.cid, obj) + got = interpreters.channel_recv(self.cid) + + self.assertEqual(got, obj) + self.assertIs(type(got), type(obj)) + # XXX Check the following in the channel tests? + #self.assertIsNot(got, obj) + + def test_singletons(self): + for obj in [None]: + with self.subTest(obj): + interpreters.channel_send(self.cid, obj) + got = interpreters.channel_recv(self.cid) + + # XXX What about between interpreters? + self.assertIs(got, obj) + + def test_types(self): + self._assert_values([ + b'spam', + 9999, + self.cid, + ]) + + def test_bytes(self): + self._assert_values(i.to_bytes(2, 'little', signed=True) + for i in range(-1, 258)) + + def test_int(self): + self._assert_values(itertools.chain(range(-1, 258), + [sys.maxsize, -sys.maxsize - 1])) + + def test_non_shareable_int(self): + ints = [ + sys.maxsize + 1, + -sys.maxsize - 2, + 2**1000, + ] + for i in ints: + with self.subTest(i): + with self.assertRaises(OverflowError): + interpreters.channel_send(self.cid, i) + + +################################## +# interpreter tests + +class ListAllTests(TestBase): + + def test_initial(self): + main = interpreters.get_main() + ids = interpreters.list_all() + self.assertEqual(ids, [main]) + + def test_after_creating(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + ids = interpreters.list_all() + self.assertEqual(ids, [main, first, second]) + + def test_after_destroying(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + interpreters.destroy(first) + ids = interpreters.list_all() + self.assertEqual(ids, [main, second]) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + cur = interpreters.get_current() + self.assertEqual(cur, main) + self.assertIsInstance(cur, interpreters.InterpreterID) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + cur = _interpreters.get_current() + print(cur) + assert isinstance(cur, _interpreters.InterpreterID) + """)) + cur = int(out.strip()) + _, expected = interpreters.list_all() + self.assertEqual(cur, expected) + self.assertNotEqual(cur, main) + + +class GetMainTests(TestBase): + + def test_from_main(self): + [expected] = interpreters.list_all() + main = interpreters.get_main() + self.assertEqual(main, expected) + self.assertIsInstance(main, interpreters.InterpreterID) + + def test_from_subinterpreter(self): + [expected] = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + main = _interpreters.get_main() + print(main) + assert isinstance(main, _interpreters.InterpreterID) + """)) + main = int(out.strip()) + self.assertEqual(main, expected) + + +class IsRunningTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(interpreters.is_running(main)) + + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interpreters.is_running(interp)) + + with _running(interp): + self.assertTrue(interpreters.is_running(interp)) + self.assertFalse(interpreters.is_running(interp)) + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + if _interpreters.is_running({interp}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interpreters.destroy(interp) + with self.assertRaises(RuntimeError): + interpreters.is_running(interp) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(-1) + + +class InterpreterIDTests(TestBase): + + def test_with_int(self): + id = interpreters.InterpreterID(10, force=True) + + self.assertEqual(int(id), 10) + + def test_coerce_id(self): + id = interpreters.InterpreterID('10', force=True) + self.assertEqual(int(id), 10) + + id = interpreters.InterpreterID(10.0, force=True) + self.assertEqual(int(id), 10) + + class Int(str): + def __init__(self, value): + self._value = value + def __int__(self): + return self._value + + id = interpreters.InterpreterID(Int(10), force=True) + self.assertEqual(int(id), 10) + + def test_bad_id(self): + for id in [-1, 'spam']: + with self.subTest(id): + with self.assertRaises(ValueError): + interpreters.InterpreterID(id) + with self.assertRaises(OverflowError): + interpreters.InterpreterID(2**64) + with self.assertRaises(TypeError): + interpreters.InterpreterID(object()) + + def test_does_not_exist(self): + id = interpreters.channel_create() + with self.assertRaises(RuntimeError): + interpreters.InterpreterID(int(id) + 1) # unforced + + def test_str(self): + id = interpreters.InterpreterID(10, force=True) + self.assertEqual(str(id), '10') + + def test_repr(self): + id = interpreters.InterpreterID(10, force=True) + self.assertEqual(repr(id), 'InterpreterID(10)') + + def test_equality(self): + id1 = interpreters.create() + id2 = interpreters.InterpreterID(int(id1)) + id3 = interpreters.create() + + self.assertTrue(id1 == id1) + self.assertTrue(id1 == id2) + self.assertTrue(id1 == int(id1)) + self.assertFalse(id1 == id3) + + self.assertFalse(id1 != id1) + self.assertFalse(id1 != id2) + self.assertTrue(id1 != id3) + + +class CreateTests(TestBase): + + def test_in_main(self): + id = interpreters.create() + self.assertIsInstance(id, interpreters.InterpreterID) + + self.assertIn(id, interpreters.list_all()) + + @unittest.skip('enable this test when working on pystate.c') + def test_unique_id(self): + seen = set() + for _ in range(100): + id = interpreters.create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_in_thread(self): + lock = threading.Lock() + id = None + def f(): + nonlocal id + id = interpreters.create() + lock.acquire() + lock.release() + + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(id, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + print(id) + assert isinstance(id, _interpreters.InterpreterID) + """)) + id2 = int(out.strip()) + + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + + def test_in_threaded_subinterpreter(self): + main, = interpreters.list_all() + id1 = interpreters.create() + id2 = None + def f(): + nonlocal id2 + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + print(id) + """)) + id2 = int(out.strip()) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {id}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {id, id2}) + + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + self.assertIn(id2, interpreters.list_all()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters.list_all()) + self.assertIn(id1, interpreters.list_all()) + self.assertIn(id3, interpreters.list_all()) + + def test_all(self): + before = set(interpreters.list_all()) + ids = set() + for _ in range(3): + id = interpreters.create() + ids.add(id) + self.assertEqual(set(interpreters.list_all()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + def test_from_current(self): + main, = interpreters.list_all() + id = interpreters.create() + script = dedent(f""" + import _xxsubinterpreters as _interpreters + try: + _interpreters.destroy({id}) + except RuntimeError: + pass + """) + + interpreters.run_string(id, script) + self.assertEqual(set(interpreters.list_all()), {main, id}) + + def test_from_sibling(self): + main, = interpreters.list_all() + id1 = interpreters.create() + id2 = interpreters.create() + script = dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.destroy({id2}) + """) + interpreters.run_string(id1, script) + + self.assertEqual(set(interpreters.list_all()), {main, id1}) + + def test_from_other_thread(self): + id = interpreters.create() + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interpreters.destroy(interp) + self.assertTrue(interpreters.is_running(interp)) + + +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + super().setUp() + self.id = interpreters.create() + self._fs = None + + def tearDown(self): + if self._fs is not None: + self._fs.close() + super().tearDown() + + @property + def fs(self): + if self._fs is None: + self._fs = FSFixture(self) + return self._fs + + def test_success(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + interpreters.run_string(self.id, script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_in_thread(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_create_thread(self): + script, file = _captured_script(""" + import threading + def f(): + print('it worked!', end='') + + t = threading.Thread(target=f) + t.start() + t.join() + """) + with file: + interpreters.run_string(self.id, script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + import tempfile + with tempfile.NamedTemporaryFile('w+') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w') as out: + out.write('{expected}') + """) + interpreters.run_string(self.id, script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + def test_already_running(self): + with _running(self.id): + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + + def test_does_not_exist(self): + id = 0 + while id in interpreters.list_all(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_script(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_script(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + @contextlib.contextmanager + def assert_run_failed(self, exctype, msg=None): + with self.assertRaises(interpreters.RunFailedError) as caught: + yield + if msg is None: + self.assertEqual(str(caught.exception).split(':')[0], + str(exctype)) + else: + self.assertEqual(str(caught.exception), + "{}: {}".format(exctype, msg)) + + def test_invalid_syntax(self): + with self.assert_run_failed(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assert_run_failed(Exception, 'spam'): + interpreters.run_string(self.id, 'raise Exception("spam")') + + def test_SystemExit(self): + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, 'raise SystemExit(42)') + + def test_sys_exit(self): + with self.assert_run_failed(SystemExit): + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + + def test_with_shared(self): + r, w = os.pipe() + + shared = { + 'spam': b'ham', + 'eggs': b'-1', + 'cheddar': None, + } + script = dedent(f""" + eggs = int(eggs) + spam = 42 + result = spam + eggs + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + self.assertIsNone(ns['cheddar']) + + def test_shared_overwrites(self): + interpreters.run_string(self.id, dedent(""" + spam = 'eggs' + ns1 = dict(vars()) + del ns1['__builtins__'] + """)) + + shared = {'spam': b'ham'} + script = dedent(f""" + ns2 = dict(vars()) + del ns2['__builtins__'] + """) + interpreters.run_string(self.id, script, shared) + + r, w = os.pipe() + script = dedent(f""" + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['ns1']['spam'], 'eggs') + self.assertEqual(ns['ns2']['spam'], b'ham') + self.assertEqual(ns['spam'], b'ham') + + def test_shared_overwrites_default_vars(self): + r, w = os.pipe() + + shared = {'__name__': b'not __main__'} + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['__name__'], b'not __main__') + + def test_main_reused(self): + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + spam = True + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + del ns, pickle, chan + """)) + with open(r, 'rb') as chan: + ns1 = pickle.load(chan) + + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + eggs = False + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """)) + with open(r, 'rb') as chan: + ns2 = pickle.load(chan) + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertIn('spam', ns2) + + def test_execution_namespace_is_main(self): + r, w = os.pipe() + + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + ns['__builtins__'] = str(ns['__builtins__']) + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + # XXX Fix this test! + @unittest.skip('blocking forever') + def test_still_running_at_exit(self): + script = dedent(f""" + from textwrap import dedent + import threading + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + def f(): + _interpreters.run_string(id, dedent(''' + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + ''')) + + t = threading.Thread(target=f) + t.start() + """) + with support.temp_dir() as dirname: + filename = script_helper.make_script(dirname, 'interp', script) + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + +################################## +# channel tests + +class ChannelIDTests(TestBase): + + def test_default_kwargs(self): + cid = interpreters._channel_id(10, force=True) + + self.assertEqual(int(cid), 10) + self.assertEqual(cid.end, 'both') + + def test_with_kwargs(self): + cid = interpreters._channel_id(10, send=True, force=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters._channel_id(10, send=True, recv=False, force=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters._channel_id(10, recv=True, force=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters._channel_id(10, recv=True, send=False, force=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters._channel_id(10, send=True, recv=True, force=True) + self.assertEqual(cid.end, 'both') + + def test_coerce_id(self): + cid = interpreters._channel_id('10', force=True) + self.assertEqual(int(cid), 10) + + cid = interpreters._channel_id(10.0, force=True) + self.assertEqual(int(cid), 10) + + class Int(str): + def __init__(self, value): + self._value = value + def __int__(self): + return self._value + + cid = interpreters._channel_id(Int(10), force=True) + self.assertEqual(int(cid), 10) + + def test_bad_id(self): + for cid in [-1, 'spam']: + with self.subTest(cid): + with self.assertRaises(ValueError): + interpreters._channel_id(cid) + with self.assertRaises(OverflowError): + interpreters._channel_id(2**64) + with self.assertRaises(TypeError): + interpreters._channel_id(object()) + + def test_bad_kwargs(self): + with self.assertRaises(ValueError): + interpreters._channel_id(10, send=False, recv=False) + + def test_does_not_exist(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters._channel_id(int(cid) + 1) # unforced + + def test_str(self): + cid = interpreters._channel_id(10, force=True) + self.assertEqual(str(cid), '10') + + def test_repr(self): + cid = interpreters._channel_id(10, force=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + cid = interpreters._channel_id(10, send=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10, send=True)') + + cid = interpreters._channel_id(10, recv=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') + + cid = interpreters._channel_id(10, send=True, recv=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + def test_equality(self): + cid1 = interpreters.channel_create() + cid2 = interpreters._channel_id(int(cid1)) + cid3 = interpreters.channel_create() + + self.assertTrue(cid1 == cid1) + self.assertTrue(cid1 == cid2) + self.assertTrue(cid1 == int(cid1)) + self.assertFalse(cid1 == cid3) + + self.assertFalse(cid1 != cid1) + self.assertFalse(cid1 != cid2) + self.assertTrue(cid1 != cid3) + + +class ChannelTests(TestBase): + + def test_create_cid(self): + cid = interpreters.channel_create() + self.assertIsInstance(cid, interpreters.ChannelID) + + def test_sequential_ids(self): + before = interpreters.channel_list_all() + id1 = interpreters.channel_create() + id2 = interpreters.channel_create() + id3 = interpreters.channel_create() + after = interpreters.channel_list_all() + + self.assertEqual(id2, int(id1) + 1) + self.assertEqual(id3, int(id2) + 1) + self.assertEqual(set(after) - set(before), {id1, id2, id3}) + + def test_ids_global(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + print(cid) + """)) + cid1 = int(out.strip()) + + id2 = interpreters.create() + out = _run_output(id2, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + print(cid) + """)) + cid2 = int(out.strip()) + + self.assertEqual(cid2, int(cid1) + 1) + + #################### + + def test_send_recv_main(self): + cid = interpreters.channel_create() + orig = b'spam' + interpreters.channel_send(cid, orig) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + orig = b'spam' + _interpreters.channel_send(cid, orig) + obj = _interpreters.channel_recv(cid) + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_interpreters(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + out = _run_output(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_send_recv_different_threads(self): + cid = interpreters.channel_create() + + def f(): + while True: + try: + obj = interpreters.channel_recv(cid) + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + interpreters.channel_send(cid, obj) + t = threading.Thread(target=f) + t.start() + + interpreters.channel_send(cid, b'spam') + t.join() + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_send_recv_different_interpreters_and_threads(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + out = None + + def f(): + nonlocal out + out = _run_output(id1, dedent(f""" + import time + import _xxsubinterpreters as _interpreters + while True: + try: + obj = _interpreters.channel_recv({cid}) + break + except _interpreters.ChannelEmptyError: + time.sleep(0.1) + assert(obj == b'spam') + _interpreters.channel_send({cid}, b'eggs') + """)) + t = threading.Thread(target=f) + t.start() + + interpreters.channel_send(cid, b'spam') + t.join() + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'eggs') + + def test_send_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_send(10, b'spam') + + def test_recv_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_recv(10) + + def test_recv_empty(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelEmptyError): + interpreters.channel_recv(cid) + + def test_run_string_arg_unresolved(self): + cid = interpreters.channel_create() + interp = interpreters.create() + + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(cid.end) + _interpreters.channel_send(cid, b'spam') + """), + dict(cid=cid.send)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') + + # XXX For now there is no high-level channel into which the + # sent channel ID can be converted... + # Note: this test caused crashes on some buildbots (bpo-33615). + @unittest.skip('disabled until high-level channels exist') + def test_run_string_arg_resolved(self): + cid = interpreters.channel_create() + cid = interpreters._channel_id(cid, _resolve=True) + interp = interpreters.create() + + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(chan.id.end) + _interpreters.channel_send(chan.id, b'spam') + """), + dict(chan=cid.send)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') + + # close + + def test_close_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + interpreters.run_string(id2, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_recv({cid}) + """)) + interpreters.channel_close(cid) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_send({cid}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id2, dedent(f""" + _interpreters.channel_send({cid}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + + def test_close_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) + + def test_close_empty(self): + tests = [ + (False, False), + (True, False), + (False, True), + (True, True), + ] + for send, recv in tests: + with self.subTest((send, recv)): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid, send=send, recv=recv) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_defaults_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + interpreters.channel_close(cid) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'eggs') + + def test_close_recv_with_unused_items_unforced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + interpreters.channel_close(cid, recv=True) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'eggs') + interpreters.channel_recv(cid) + interpreters.channel_recv(cid) + interpreters.channel_close(cid, recv=True) + + def test_close_send_with_unused_items_unforced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, send=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + interpreters.channel_recv(cid) + interpreters.channel_recv(cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_both_with_unused_items_unforced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + interpreters.channel_close(cid, recv=True, send=True) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'eggs') + interpreters.channel_recv(cid) + interpreters.channel_recv(cid) + interpreters.channel_close(cid, recv=True) + + def test_close_recv_with_unused_items_forced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, recv=True, force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_send_with_unused_items_forced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, send=True, force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_both_with_unused_items_forced(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, send=True, recv=True, force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_close({cid}, force=True) + """)) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) + + def test_close_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid, force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + +class ChannelReleaseTests(TestBase): + + # XXX Add more test coverage a la the tests for close(). + + """ + - main / interp / other + - run in: current thread / new thread / other thread / different threads + - end / opposite + - force / no force + - used / not used (associated / not associated) + - empty / emptied / never emptied / partly emptied + - closed / not closed + - released / not released + - creator (interp) / other + - associated interpreter not running + - associated interpreter destroyed + """ + + """ + use + pre-release + release + after + check + """ + + """ + release in: main, interp1 + creator: same, other (incl. interp2) + + use: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all + pre-release: None,send,recv,both in None,same,other(incl. interp2),same+other(incl. interp2),all + pre-release forced: None,send,recv,both in None,same,other(incl. interp2),same+other(incl. interp2),all + + release: same + release forced: same + + use after: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all + release after: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all + check released: send/recv for same/other(incl. interp2) + check closed: send/recv for same/other(incl. interp2) + """ + + def test_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_release(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + out = _run_output(id2, dedent(f""" + import _xxsubinterpreters as _interpreters + obj = _interpreters.channel_recv({cid}) + _interpreters.channel_release({cid}) + print(repr(obj)) + """)) + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_release({cid}) + """)) + + self.assertEqual(out.strip(), "b'spam'") + + def test_no_kwargs(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_release(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_release(cid, send=True, recv=True) + + def test_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_release(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_release({cid}) + """)) + obj = interpreters.channel_recv(cid) + interpreters.channel_release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + self.assertEqual(obj, b'spam') + + def test_close_if_unassociated(self): + # XXX Something's not right with this test... + cid = interpreters.channel_create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + obj = _interpreters.channel_send({cid}, b'spam') + _interpreters.channel_release({cid}) + """)) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_partially(self): + # XXX Is partial close too weird/confusing? + cid = interpreters.channel_create() + interpreters.channel_send(cid, None) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'spam') + interpreters.channel_release(cid, send=True) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_release(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + +class ChannelCloseFixture(namedtuple('ChannelCloseFixture', + 'end interp other extra creator')): + + # Set this to True to avoid creating interpreters, e.g. when + # scanning through test permutations without running them. + QUICK = False + + def __new__(cls, end, interp, other, extra, creator): + assert end in ('send', 'recv') + if cls.QUICK: + known = {} + else: + interp = Interpreter.from_raw(interp) + other = Interpreter.from_raw(other) + extra = Interpreter.from_raw(extra) + known = { + interp.name: interp, + other.name: other, + extra.name: extra, + } + if not creator: + creator = 'same' + self = super().__new__(cls, end, interp, other, extra, creator) + self._prepped = set() + self._state = ChannelState() + self._known = known + return self + + @property + def state(self): + return self._state + + @property + def cid(self): + try: + return self._cid + except AttributeError: + creator = self._get_interpreter(self.creator) + self._cid = self._new_channel(creator) + return self._cid + + def get_interpreter(self, interp): + interp = self._get_interpreter(interp) + self._prep_interpreter(interp) + return interp + + def expect_closed_error(self, end=None): + if end is None: + end = self.end + if end == 'recv' and self.state.closed == 'send': + return False + return bool(self.state.closed) + + def prep_interpreter(self, interp): + self._prep_interpreter(interp) + + def record_action(self, action, result): + self._state = result + + def clean_up(self): + clean_up_interpreters() + clean_up_channels() + + # internal methods + + def _new_channel(self, creator): + if creator.name == 'main': + return interpreters.channel_create() + else: + ch = interpreters.channel_create() + run_interp(creator.id, f""" + import _xxsubinterpreters + cid = _xxsubinterpreters.channel_create() + # We purposefully send back an int to avoid tying the + # channel to the other interpreter. + _xxsubinterpreters.channel_send({ch}, int(cid)) + del _xxsubinterpreters + """) + self._cid = interpreters.channel_recv(ch) + return self._cid + + def _get_interpreter(self, interp): + if interp in ('same', 'interp'): + return self.interp + elif interp == 'other': + return self.other + elif interp == 'extra': + return self.extra + else: + name = interp + try: + interp = self._known[name] + except KeyError: + interp = self._known[name] = Interpreter(name) + return interp + + def _prep_interpreter(self, interp): + if interp.id in self._prepped: + return + self._prepped.add(interp.id) + if interp.name == 'main': + return + run_interp(interp.id, f""" + import _xxsubinterpreters as interpreters + import test.test__xxsubinterpreters as helpers + ChannelState = helpers.ChannelState + try: + cid + except NameError: + cid = interpreters._channel_id({self.cid}) + """) + + +@unittest.skip('these tests take several hours to run') +class ExhaustiveChannelTests(TestBase): + + """ + - main / interp / other + - run in: current thread / new thread / other thread / different threads + - end / opposite + - force / no force + - used / not used (associated / not associated) + - empty / emptied / never emptied / partly emptied + - closed / not closed + - released / not released + - creator (interp) / other + - associated interpreter not running + - associated interpreter destroyed + + - close after unbound + """ + + """ + use + pre-close + close + after + check + """ + + """ + close in: main, interp1 + creator: same, other, extra + + use: None,send,recv,send/recv in None,same,other,same+other,all + pre-close: None,send,recv in None,same,other,same+other,all + pre-close forced: None,send,recv in None,same,other,same+other,all + + close: same + close forced: same + + use after: None,send,recv,send/recv in None,same,other,extra,same+other,all + close after: None,send,recv,send/recv in None,same,other,extra,same+other,all + check closed: send/recv for same/other(incl. interp2) + """ + + def iter_action_sets(self): + # - used / not used (associated / not associated) + # - empty / emptied / never emptied / partly emptied + # - closed / not closed + # - released / not released + + # never used + yield [] + + # only pre-closed (and possible used after) + for closeactions in self._iter_close_action_sets('same', 'other'): + yield closeactions + for postactions in self._iter_post_close_action_sets(): + yield closeactions + postactions + for closeactions in self._iter_close_action_sets('other', 'extra'): + yield closeactions + for postactions in self._iter_post_close_action_sets(): + yield closeactions + postactions + + # used + for useactions in self._iter_use_action_sets('same', 'other'): + yield useactions + for closeactions in self._iter_close_action_sets('same', 'other'): + actions = useactions + closeactions + yield actions + for postactions in self._iter_post_close_action_sets(): + yield actions + postactions + for closeactions in self._iter_close_action_sets('other', 'extra'): + actions = useactions + closeactions + yield actions + for postactions in self._iter_post_close_action_sets(): + yield actions + postactions + for useactions in self._iter_use_action_sets('other', 'extra'): + yield useactions + for closeactions in self._iter_close_action_sets('same', 'other'): + actions = useactions + closeactions + yield actions + for postactions in self._iter_post_close_action_sets(): + yield actions + postactions + for closeactions in self._iter_close_action_sets('other', 'extra'): + actions = useactions + closeactions + yield actions + for postactions in self._iter_post_close_action_sets(): + yield actions + postactions + + def _iter_use_action_sets(self, interp1, interp2): + interps = (interp1, interp2) + + # only recv end used + yield [ + ChannelAction('use', 'recv', interp1), + ] + yield [ + ChannelAction('use', 'recv', interp2), + ] + yield [ + ChannelAction('use', 'recv', interp1), + ChannelAction('use', 'recv', interp2), + ] + + # never emptied + yield [ + ChannelAction('use', 'send', interp1), + ] + yield [ + ChannelAction('use', 'send', interp2), + ] + yield [ + ChannelAction('use', 'send', interp1), + ChannelAction('use', 'send', interp2), + ] + + # partially emptied + for interp1 in interps: + for interp2 in interps: + for interp3 in interps: + yield [ + ChannelAction('use', 'send', interp1), + ChannelAction('use', 'send', interp2), + ChannelAction('use', 'recv', interp3), + ] + + # fully emptied + for interp1 in interps: + for interp2 in interps: + for interp3 in interps: + for interp4 in interps: + yield [ + ChannelAction('use', 'send', interp1), + ChannelAction('use', 'send', interp2), + ChannelAction('use', 'recv', interp3), + ChannelAction('use', 'recv', interp4), + ] + + def _iter_close_action_sets(self, interp1, interp2): + ends = ('recv', 'send') + interps = (interp1, interp2) + for force in (True, False): + op = 'force-close' if force else 'close' + for interp in interps: + for end in ends: + yield [ + ChannelAction(op, end, interp), + ] + for recvop in ('close', 'force-close'): + for sendop in ('close', 'force-close'): + for recv in interps: + for send in interps: + yield [ + ChannelAction(recvop, 'recv', recv), + ChannelAction(sendop, 'send', send), + ] + + def _iter_post_close_action_sets(self): + for interp in ('same', 'extra', 'other'): + yield [ + ChannelAction('use', 'recv', interp), + ] + yield [ + ChannelAction('use', 'send', interp), + ] + + def run_actions(self, fix, actions): + for action in actions: + self.run_action(fix, action) + + def run_action(self, fix, action, *, hideclosed=True): + end = action.resolve_end(fix.end) + interp = action.resolve_interp(fix.interp, fix.other, fix.extra) + fix.prep_interpreter(interp) + if interp.name == 'main': + result = run_action( + fix.cid, + action.action, + end, + fix.state, + hideclosed=hideclosed, + ) + fix.record_action(action, result) + else: + _cid = interpreters.channel_create() + run_interp(interp.id, f""" + result = helpers.run_action( + {fix.cid}, + {repr(action.action)}, + {repr(end)}, + {repr(fix.state)}, + hideclosed={hideclosed}, + ) + interpreters.channel_send({_cid}, result.pending.to_bytes(1, 'little')) + interpreters.channel_send({_cid}, b'X' if result.closed else b'') + """) + result = ChannelState( + pending=int.from_bytes(interpreters.channel_recv(_cid), 'little'), + closed=bool(interpreters.channel_recv(_cid)), + ) + fix.record_action(action, result) + + def iter_fixtures(self): + # XXX threads? + interpreters = [ + ('main', 'interp', 'extra'), + ('interp', 'main', 'extra'), + ('interp1', 'interp2', 'extra'), + ('interp1', 'interp2', 'main'), + ] + for interp, other, extra in interpreters: + for creator in ('same', 'other', 'creator'): + for end in ('send', 'recv'): + yield ChannelCloseFixture(end, interp, other, extra, creator) + + def _close(self, fix, *, force): + op = 'force-close' if force else 'close' + close = ChannelAction(op, fix.end, 'same') + if not fix.expect_closed_error(): + self.run_action(fix, close, hideclosed=False) + else: + with self.assertRaises(interpreters.ChannelClosedError): + self.run_action(fix, close, hideclosed=False) + + def _assert_closed_in_interp(self, fix, interp=None): + if interp is None or interp.name == 'main': + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(fix.cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(fix.cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(fix.cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(fix.cid, force=True) + else: + run_interp(interp.id, f""" + with helpers.expect_channel_closed(): + interpreters.channel_recv(cid) + """) + run_interp(interp.id, f""" + with helpers.expect_channel_closed(): + interpreters.channel_send(cid, b'spam') + """) + run_interp(interp.id, f""" + with helpers.expect_channel_closed(): + interpreters.channel_close(cid) + """) + run_interp(interp.id, f""" + with helpers.expect_channel_closed(): + interpreters.channel_close(cid, force=True) + """) + + def _assert_closed(self, fix): + self.assertTrue(fix.state.closed) + + for _ in range(fix.state.pending): + interpreters.channel_recv(fix.cid) + self._assert_closed_in_interp(fix) + + for interp in ('same', 'other'): + interp = fix.get_interpreter(interp) + if interp.name == 'main': + continue + self._assert_closed_in_interp(fix, interp) + + interp = fix.get_interpreter('fresh') + self._assert_closed_in_interp(fix, interp) + + def _iter_close_tests(self, verbose=False): + i = 0 + for actions in self.iter_action_sets(): + print() + for fix in self.iter_fixtures(): + i += 1 + if i > 1000: + return + if verbose: + if (i - 1) % 6 == 0: + print() + print(i, fix, '({} actions)'.format(len(actions))) + else: + if (i - 1) % 6 == 0: + print(' ', end='') + print('.', end=''); sys.stdout.flush() + yield i, fix, actions + if verbose: + print('---') + print() + + # This is useful for scanning through the possible tests. + def _skim_close_tests(self): + ChannelCloseFixture.QUICK = True + for i, fix, actions in self._iter_close_tests(): + pass + + def test_close(self): + for i, fix, actions in self._iter_close_tests(): + with self.subTest('{} {} {}'.format(i, fix, actions)): + fix.prep_interpreter(fix.interp) + self.run_actions(fix, actions) + + self._close(fix, force=False) + + self._assert_closed(fix) + # XXX Things slow down if we have too many interpreters. + fix.clean_up() + + def test_force_close(self): + for i, fix, actions in self._iter_close_tests(): + with self.subTest('{} {} {}'.format(i, fix, actions)): + fix.prep_interpreter(fix.interp) + self.run_actions(fix, actions) + + self._close(fix, force=True) + + self._assert_closed(fix) + # XXX Things slow down if we have too many interpreters. + fix.clean_up() + + +if __name__ == '__main__': + unittest.main() From 331be62729bad0037af12e484fe693685d22495b Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 10 Sep 2019 12:54:38 +0000 Subject: [PATCH 02/20] create stdlib interpreters module and _interpreters internal module --- Doc/library/interpreters.rst | 9 +++++ Lib/interpreters.py | 0 Lib/test/test__xxsubinterpreters.py | 56 ++++++++++++++--------------- Modules/_xxsubinterpretersmodule.c | 18 +++++----- PC/config.c | 4 +-- setup.py | 2 +- 6 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 Doc/library/interpreters.rst create mode 100644 Lib/interpreters.py diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst new file mode 100644 index 00000000000000..bb9bce2de0f42d --- /dev/null +++ b/Doc/library/interpreters.rst @@ -0,0 +1,9 @@ +:mod:`interpreters` --- High-level Sub-interpreters Module +========================================================== + +.. module:: interpreters + :synopsis: High-level Sub-interpreters Module. + +**Source code:** :source:`Lib/interpreters.py` + +-------------- \ No newline at end of file diff --git a/Lib/interpreters.py b/Lib/interpreters.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 78b2030a1f6d8b..20e6b0419ff411 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -13,7 +13,7 @@ from test.support import script_helper -interpreters = support.import_module('_xxsubinterpreters') +interpreters = support.import_module('_interpreters') ################################## @@ -446,7 +446,7 @@ def test_subinterpreter(self): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters cur = _interpreters.get_current() print(cur) assert isinstance(cur, _interpreters.InterpreterID) @@ -469,7 +469,7 @@ def test_from_subinterpreter(self): [expected] = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters main = _interpreters.get_main() print(main) assert isinstance(main, _interpreters.InterpreterID) @@ -495,7 +495,7 @@ def test_subinterpreter(self): def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters if _interpreters.is_running({interp}): print(True) else: @@ -616,7 +616,7 @@ def test_in_subinterpreter(self): main, = interpreters.list_all() id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) assert isinstance(id, _interpreters.InterpreterID) @@ -632,7 +632,7 @@ def test_in_threaded_subinterpreter(self): def f(): nonlocal id2 out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) """)) @@ -726,7 +726,7 @@ def test_from_current(self): main, = interpreters.list_all() id = interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters try: _interpreters.destroy({id}) except RuntimeError: @@ -741,7 +741,7 @@ def test_from_sibling(self): id1 = interpreters.create() id2 = interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.destroy({id2}) """) interpreters.run_string(id1, script) @@ -1057,7 +1057,7 @@ def test_still_running_at_exit(self): script = dedent(f""" from textwrap import dedent import threading - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() def f(): _interpreters.run_string(id, dedent(''' @@ -1191,7 +1191,7 @@ def test_sequential_ids(self): def test_ids_global(self): id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters cid = _interpreters.channel_create() print(cid) """)) @@ -1199,7 +1199,7 @@ def test_ids_global(self): id2 = interpreters.create() out = _run_output(id2, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters cid = _interpreters.channel_create() print(cid) """)) @@ -1221,7 +1221,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters cid = _interpreters.channel_create() orig = b'spam' _interpreters.channel_send(cid, orig) @@ -1234,7 +1234,7 @@ def test_send_recv_different_interpreters(self): cid = interpreters.channel_create() id1 = interpreters.create() out = _run_output(id1, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_send({cid}, b'spam') """)) obj = interpreters.channel_recv(cid) @@ -1270,7 +1270,7 @@ def f(): nonlocal out out = _run_output(id1, dedent(f""" import time - import _xxsubinterpreters as _interpreters + import _interpreters while True: try: obj = _interpreters.channel_recv({cid}) @@ -1307,7 +1307,7 @@ def test_run_string_arg_unresolved(self): interp = interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters print(cid.end) _interpreters.channel_send(cid, b'spam') """), @@ -1327,7 +1327,7 @@ def test_run_string_arg_resolved(self): interp = interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters print(chan.id.end) _interpreters.channel_send(chan.id, b'spam') """), @@ -1355,11 +1355,11 @@ def test_close_multiple_users(self): id1 = interpreters.create() id2 = interpreters.create() interpreters.run_string(id1, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_send({cid}, b'spam') """)) interpreters.run_string(id2, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_recv({cid}) """)) interpreters.channel_close(cid) @@ -1498,7 +1498,7 @@ def test_close_by_unassociated_interp(self): interpreters.channel_send(cid, b'spam') interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_close({cid}, force=True) """)) with self.assertRaises(interpreters.ChannelClosedError): @@ -1579,11 +1579,11 @@ def test_multiple_users(self): id1 = interpreters.create() id2 = interpreters.create() interpreters.run_string(id1, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_send({cid}, b'spam') """)) out = _run_output(id2, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters obj = _interpreters.channel_recv({cid}) _interpreters.channel_release({cid}) print(repr(obj)) @@ -1637,7 +1637,7 @@ def test_by_unassociated_interp(self): interpreters.channel_send(cid, b'spam') interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.channel_release({cid}) """)) obj = interpreters.channel_recv(cid) @@ -1652,7 +1652,7 @@ def test_close_if_unassociated(self): cid = interpreters.channel_create() interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters obj = _interpreters.channel_send({cid}, b'spam') _interpreters.channel_release({cid}) """)) @@ -1756,12 +1756,12 @@ def _new_channel(self, creator): else: ch = interpreters.channel_create() run_interp(creator.id, f""" - import _xxsubinterpreters - cid = _xxsubinterpreters.channel_create() + import _interpreters + cid = _interpreters.channel_create() # We purposefully send back an int to avoid tying the # channel to the other interpreter. - _xxsubinterpreters.channel_send({ch}, int(cid)) - del _xxsubinterpreters + _interpreters.channel_send({ch}, int(cid)) + del _interpreters """) self._cid = interpreters.channel_recv(ch) return self._cid @@ -1788,7 +1788,7 @@ def _prep_interpreter(self, interp): if interp.name == 'main': return run_interp(interp.id, f""" - import _xxsubinterpreters as interpreters + import _interpreters as interpreters import test.test__xxsubinterpreters as helpers ChannelState = helpers.ChannelState try: diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 19d98fd9693446..552d96a977f2a9 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -293,7 +293,7 @@ channel_exceptions_init(PyObject *ns) // XXX Move the exceptions into per-module memory? // A channel-related operation failed. - ChannelError = PyErr_NewException("_xxsubinterpreters.ChannelError", + ChannelError = PyErr_NewException("_interpreters.ChannelError", PyExc_RuntimeError, NULL); if (ChannelError == NULL) { return -1; @@ -304,7 +304,7 @@ channel_exceptions_init(PyObject *ns) // An operation tried to use a channel that doesn't exist. ChannelNotFoundError = PyErr_NewException( - "_xxsubinterpreters.ChannelNotFoundError", ChannelError, NULL); + "_interpreters.ChannelNotFoundError", ChannelError, NULL); if (ChannelNotFoundError == NULL) { return -1; } @@ -314,7 +314,7 @@ channel_exceptions_init(PyObject *ns) // An operation tried to use a closed channel. ChannelClosedError = PyErr_NewException( - "_xxsubinterpreters.ChannelClosedError", ChannelError, NULL); + "_interpreters.ChannelClosedError", ChannelError, NULL); if (ChannelClosedError == NULL) { return -1; } @@ -324,7 +324,7 @@ channel_exceptions_init(PyObject *ns) // An operation tried to pop from an empty channel. ChannelEmptyError = PyErr_NewException( - "_xxsubinterpreters.ChannelEmptyError", ChannelError, NULL); + "_interpreters.ChannelEmptyError", ChannelError, NULL); if (ChannelEmptyError == NULL) { return -1; } @@ -334,7 +334,7 @@ channel_exceptions_init(PyObject *ns) // An operation tried to close a non-empty channel. ChannelNotEmptyError = PyErr_NewException( - "_xxsubinterpreters.ChannelNotEmptyError", ChannelError, NULL); + "_interpreters.ChannelNotEmptyError", ChannelError, NULL); if (ChannelNotEmptyError == NULL) { return -1; } @@ -1736,7 +1736,7 @@ PyDoc_STRVAR(channelid_doc, static PyTypeObject ChannelIDtype = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "_xxsubinterpreters.ChannelID", /* tp_name */ + "_interpreters.ChannelID", /* tp_name */ sizeof(channelid), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)channelid_dealloc, /* tp_dealloc */ @@ -1793,7 +1793,7 @@ interp_exceptions_init(PyObject *ns) if (RunFailedError == NULL) { // An uncaught exception came out of interp_run_string(). - RunFailedError = PyErr_NewException("_xxsubinterpreters.RunFailedError", + RunFailedError = PyErr_NewException("_interpreters.RunFailedError", PyExc_RuntimeError, NULL); if (RunFailedError == NULL) { return -1; @@ -2519,7 +2519,7 @@ The 'interpreters' module provides a more convenient interface."); static struct PyModuleDef interpretersmodule = { PyModuleDef_HEAD_INIT, - "_xxsubinterpreters", /* m_name */ + "_interpreters", /* m_name */ module_doc, /* m_doc */ -1, /* m_size */ module_functions, /* m_methods */ @@ -2531,7 +2531,7 @@ static struct PyModuleDef interpretersmodule = { PyMODINIT_FUNC -PyInit__xxsubinterpreters(void) +PyInit__interpreters(void) { if (_init_globals() != 0) { return NULL; diff --git a/PC/config.c b/PC/config.c index 8eaeb31c9b934b..38d9506ffdd23e 100644 --- a/PC/config.c +++ b/PC/config.c @@ -35,7 +35,7 @@ extern PyObject* PyInit__codecs(void); extern PyObject* PyInit__weakref(void); /* XXX: These two should really be extracted to standalone extensions. */ extern PyObject* PyInit_xxsubtype(void); -extern PyObject* PyInit__xxsubinterpreters(void); +extern PyObject* PyInit__interpreters(void); extern PyObject* PyInit__random(void); extern PyObject* PyInit_itertools(void); extern PyObject* PyInit__collections(void); @@ -133,7 +133,7 @@ struct _inittab _PyImport_Inittab[] = { {"_json", PyInit__json}, {"xxsubtype", PyInit_xxsubtype}, - {"_xxsubinterpreters", PyInit__xxsubinterpreters}, + {"_interpreters", PyInit__interpreters}, #ifdef _Py_HAVE_ZLIB {"zlib", PyInit_zlib}, #endif diff --git a/setup.py b/setup.py index 02f523c42d355f..5243b41a3a828a 100644 --- a/setup.py +++ b/setup.py @@ -827,7 +827,7 @@ def detect_simple_extensions(self): self.add(Extension('syslog', ['syslogmodule.c'])) # Python interface to subinterpreter C-API. - self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'])) + self.add(Extension('_interpreters', ['_xxsubinterpretersmodule.c'])) # # Here ends the simple stuff. From here on, modules need certain From 3f2f2f4f862fde1455fdc178265172040cd7f7e8 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 10 Sep 2019 16:25:02 +0000 Subject: [PATCH 03/20] Global initial Sub-Interpreters functions --- Doc/library/interpreters.rst | 38 +- Lib/interpreters.py | 59 + Lib/test/test_interpreters.py | 2037 +-------------------------------- 3 files changed, 103 insertions(+), 2031 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index bb9bce2de0f42d..28ab95c370ce66 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -2,8 +2,42 @@ ========================================================== .. module:: interpreters - :synopsis: High-level Sub-interpreters Module. + :synopsis: High-level Sub-Interpreters Module. **Source code:** :source:`Lib/interpreters.py` --------------- \ No newline at end of file +-------------- + +This module constructs higher-level interpreters interfaces on top of the lower +level :mod:`_interpreters` module. + +.. versionchanged:: 3.9 + + +This module defines the following functions: + + +.. function:: create() + + Create a new interpreter and return a unique generated ID. + +.. function:: list_all() + + Return a list containing the ID of every existing interpreter. + +.. function:: get_current() + + Return the ID of the currently running interpreter. + +.. function:: destroy(id) + + Destroy the interpreter whose ID is *id*. + +.. function:: get_main() + + Return the ID of the main interpreter. + +.. function:: run_string() + + Execute the provided string in the identified interpreter. + See `PyRun_SimpleStrings`. diff --git a/Lib/interpreters.py b/Lib/interpreters.py index e69de29bb2d1d6..95537bba6cbdae 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -0,0 +1,59 @@ +"""Sub-interpreters High Level Module.""" + +import _interpreters + +__all__ = ['create', 'list_all', 'get_current'] + +# Rename so that "from interpreters import *" is safe +_list_all = _interpreters.list_all +_get_current = _interpreters.get_current +_get_main = _interpreters.get_main +_run_string = _interpreters.run_string + +# Global API functions + +def create(): + """ create() -> Interpreter + + Create a new interpreter and return a unique generated ID. + """ + return _interpreters.create() + +def list_all(): + """list_all() -> [Interpreter] + + Return a list containing the ID of every existing interpreter. + """ + return _list_all() + +def get_current(): + """get_current() -> Interpreter + + Return the ID of the currently running interpreter. + """ + return _get_current() + +def get_main(): + """get_main() -> ID + + Return the ID of the main interpreter. + """ + + return _get_main() + +def destroy(id): + """destroy(id) -> None + + Destroy the identified interpreter. + """ + + return _interpreters.destroy(id) + +def run_string(id, script, shared): + """run_string(id, script, shared) -> None + + Execute the provided string in the identified interpreter. + See PyRun_SimpleStrings. + """ + + return _run_string(id, script, shared) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 65c6e77d69d1ee..32964ec435f4c0 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,4 +1,5 @@ import interpreters +import _interpreters #remove when all methods are implemented from collections import namedtuple import contextlib import itertools @@ -13,16 +14,6 @@ from test import support from test.support import script_helper - -################################## -# helpers - -def powerset(*sets): - return itertools.chain.from_iterable( - combinations(sets, r) - for r in range(len(sets)+1)) - - def _captured_script(script): r, w = os.pipe() indented = script.replace('\n', '\n ') @@ -34,243 +25,6 @@ def _captured_script(script): """) return wrapped, open(r) - -def _run_output(interp, request, shared=None): - script, rpipe = _captured_script(request) - with rpipe: - interpreters.run_string(interp, script, shared) - return rpipe.read() - - -@contextlib.contextmanager -def _running(interp): - r, w = os.pipe() - def run(): - interpreters.run_string(interp, dedent(f""" - # wait for "signal" - with open({r}) as rpipe: - rpipe.read() - """)) - - t = threading.Thread(target=run) - t.start() - - yield - - with open(w, 'w') as spipe: - spipe.write('done') - t.join() - - -#@contextmanager -#def run_threaded(id, source, **shared): -# def run(): -# run_interp(id, source, **shared) -# t = threading.Thread(target=run) -# t.start() -# yield -# t.join() - - -def run_interp(id, source, **shared): - _run_interp(id, source, shared) - - -def _run_interp(id, source, shared, _mainns={}): - source = dedent(source) - main = interpreters.get_main() - if main == id: - if interpreters.get_current() != main: - raise RuntimeError - # XXX Run a func? - exec(source, _mainns) - else: - interpreters.run_string(id, source, shared) - - -def run_interp_threaded(id, source, **shared): - def run(): - _run(id, source, shared) - t = threading.Thread(target=run) - t.start() - t.join() - - -class Interpreter(namedtuple('Interpreter', 'name id')): - - @classmethod - def from_raw(cls, raw): - if isinstance(raw, cls): - return raw - elif isinstance(raw, str): - return cls(raw) - else: - raise NotImplementedError - - def __new__(cls, name=None, id=None): - main = interpreters.get_main() - if id == main: - if not name: - name = 'main' - elif name != 'main': - raise ValueError( - 'name mismatch (expected "main", got "{}")'.format(name)) - id = main - elif id is not None: - if not name: - name = 'interp' - elif name == 'main': - raise ValueError('name mismatch (unexpected "main")') - if not isinstance(id, interpreters.InterpreterID): - id = interpreters.InterpreterID(id) - elif not name or name == 'main': - name = 'main' - id = main - else: - id = interpreters.create() - self = super().__new__(cls, name, id) - return self - - -# XXX expect_channel_closed() is unnecessary once we improve exc propagation. - -@contextlib.contextmanager -def expect_channel_closed(): - try: - yield - except interpreters.ChannelClosedError: - pass - else: - assert False, 'channel not closed' - - -class ChannelAction(namedtuple('ChannelAction', 'action end interp')): - - def __new__(cls, action, end=None, interp=None): - if not end: - end = 'both' - if not interp: - interp = 'main' - self = super().__new__(cls, action, end, interp) - return self - - def __init__(self, *args, **kwargs): - if self.action == 'use': - if self.end not in ('same', 'opposite', 'send', 'recv'): - raise ValueError(self.end) - elif self.action in ('close', 'force-close'): - if self.end not in ('both', 'same', 'opposite', 'send', 'recv'): - raise ValueError(self.end) - else: - raise ValueError(self.action) - if self.interp not in ('main', 'same', 'other', 'extra'): - raise ValueError(self.interp) - - def resolve_end(self, end): - if self.end == 'same': - return end - elif self.end == 'opposite': - return 'recv' if end == 'send' else 'send' - else: - return self.end - - def resolve_interp(self, interp, other, extra): - if self.interp == 'same': - return interp - elif self.interp == 'other': - if other is None: - raise RuntimeError - return other - elif self.interp == 'extra': - if extra is None: - raise RuntimeError - return extra - elif self.interp == 'main': - if interp.name == 'main': - return interp - elif other and other.name == 'main': - return other - else: - raise RuntimeError - # Per __init__(), there aren't any others. - - -class ChannelState(namedtuple('ChannelState', 'pending closed')): - - def __new__(cls, pending=0, *, closed=False): - self = super().__new__(cls, pending, closed) - return self - - def incr(self): - return type(self)(self.pending + 1, closed=self.closed) - - def decr(self): - return type(self)(self.pending - 1, closed=self.closed) - - def close(self, *, force=True): - if self.closed: - if not force or self.pending == 0: - return self - return type(self)(0 if force else self.pending, closed=True) - - -def run_action(cid, action, end, state, *, hideclosed=True): - if state.closed: - if action == 'use' and end == 'recv' and state.pending: - expectfail = False - else: - expectfail = True - else: - expectfail = False - - try: - result = _run_action(cid, action, end, state) - except interpreters.ChannelClosedError: - if not hideclosed and not expectfail: - raise - result = state.close() - else: - if expectfail: - raise ... # XXX - return result - - -def _run_action(cid, action, end, state): - if action == 'use': - if end == 'send': - interpreters.channel_send(cid, b'spam') - return state.incr() - elif end == 'recv': - if not state.pending: - try: - interpreters.channel_recv(cid) - except interpreters.ChannelEmptyError: - return state - else: - raise Exception('expected ChannelEmptyError') - else: - interpreters.channel_recv(cid) - return state.decr() - else: - raise ValueError(end) - elif action == 'close': - kwargs = {} - if end in ('recv', 'send'): - kwargs[end] = True - interpreters.channel_close(cid, **kwargs) - return state.close() - elif action == 'force-close': - kwargs = { - 'force': True, - } - if end in ('recv', 'send'): - kwargs[end] = True - interpreters.channel_close(cid, **kwargs) - return state.close(force=True) - else: - raise ValueError(action) - - def clean_up_interpreters(): for id in interpreters.list_all(): if id == 0: # main @@ -281,133 +35,11 @@ def clean_up_interpreters(): pass # already destroyed -def clean_up_channels(): - for cid in interpreters.channel_list_all(): - try: - interpreters.channel_destroy(cid) - except interpreters.ChannelNotFoundError: - pass # already destroyed - - class TestBase(unittest.TestCase): def tearDown(self): clean_up_interpreters() - clean_up_channels() - - -################################## -# misc. tests - -class IsShareableTests(unittest.TestCase): - - def test_default_shareables(self): - shareables = [ - # singletons - None, - # builtin objects - b'spam', - 'spam', - 10, - -10, - ] - for obj in shareables: - with self.subTest(obj): - self.assertTrue( - interpreters.is_shareable(obj)) - - def test_not_shareable(self): - class Cheese: - def __init__(self, name): - self.name = name - def __str__(self): - return self.name - - class SubBytes(bytes): - """A subclass of a shareable type.""" - - not_shareables = [ - # singletons - True, - False, - NotImplemented, - ..., - # builtin types and objects - type, - object, - object(), - Exception(), - 100.0, - # user-defined types and objects - Cheese, - Cheese('Wensleydale'), - SubBytes(b'spam'), - ] - for obj in not_shareables: - with self.subTest(repr(obj)): - self.assertFalse( - interpreters.is_shareable(obj)) - - -class ShareableTypeTests(unittest.TestCase): - - def setUp(self): - super().setUp() - self.cid = interpreters.channel_create() - - def tearDown(self): - interpreters.channel_destroy(self.cid) - super().tearDown() - def _assert_values(self, values): - for obj in values: - with self.subTest(obj): - interpreters.channel_send(self.cid, obj) - got = interpreters.channel_recv(self.cid) - - self.assertEqual(got, obj) - self.assertIs(type(got), type(obj)) - # XXX Check the following in the channel tests? - #self.assertIsNot(got, obj) - - def test_singletons(self): - for obj in [None]: - with self.subTest(obj): - interpreters.channel_send(self.cid, obj) - got = interpreters.channel_recv(self.cid) - - # XXX What about between interpreters? - self.assertIs(got, obj) - - def test_types(self): - self._assert_values([ - b'spam', - 9999, - self.cid, - ]) - - def test_bytes(self): - self._assert_values(i.to_bytes(2, 'little', signed=True) - for i in range(-1, 258)) - - def test_int(self): - self._assert_values(itertools.chain(range(-1, 258), - [sys.maxsize, -sys.maxsize - 1])) - - def test_non_shareable_int(self): - ints = [ - sys.maxsize + 1, - -sys.maxsize - 2, - 2**1000, - ] - for i in ints: - with self.subTest(i): - with self.assertRaises(OverflowError): - interpreters.channel_send(self.cid, i) - - -################################## -# interpreter tests class ListAllTests(TestBase): @@ -416,1704 +48,51 @@ def test_initial(self): ids = interpreters.list_all() self.assertEqual(ids, [main]) - def test_after_creating(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - ids = interpreters.list_all() - self.assertEqual(ids, [main, first, second]) - - def test_after_destroying(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - interpreters.destroy(first) - ids = interpreters.list_all() - self.assertEqual(ids, [main, second]) - class GetCurrentTests(TestBase): - def test_main(self): + def test_get_current(self): main = interpreters.get_main() cur = interpreters.get_current() self.assertEqual(cur, main) - self.assertIsInstance(cur, interpreters.InterpreterID) - - def test_subinterpreter(self): - main = interpreters.get_main() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - cur = _interpreters.get_current() - print(cur) - assert isinstance(cur, _interpreters.InterpreterID) - """)) - cur = int(out.strip()) - _, expected = interpreters.list_all() - self.assertEqual(cur, expected) - self.assertNotEqual(cur, main) class GetMainTests(TestBase): - def test_from_main(self): + def test_get_main(self): [expected] = interpreters.list_all() main = interpreters.get_main() self.assertEqual(main, expected) - self.assertIsInstance(main, interpreters.InterpreterID) - - def test_from_subinterpreter(self): - [expected] = interpreters.list_all() - interp = interpreters.create() - out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - main = _interpreters.get_main() - print(main) - assert isinstance(main, _interpreters.InterpreterID) - """)) - main = int(out.strip()) - self.assertEqual(main, expected) - - -class IsRunningTests(TestBase): - - def test_main(self): - main = interpreters.get_main() - self.assertTrue(interpreters.is_running(main)) - - def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interpreters.is_running(interp)) - - with _running(interp): - self.assertTrue(interpreters.is_running(interp)) - self.assertFalse(interpreters.is_running(interp)) - - def test_from_subinterpreter(self): - interp = interpreters.create() - out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - if _interpreters.is_running({interp}): - print(True) - else: - print(False) - """)) - self.assertEqual(out.strip(), 'True') - - def test_already_destroyed(self): - interp = interpreters.create() - interpreters.destroy(interp) - with self.assertRaises(RuntimeError): - interpreters.is_running(interp) - - def test_does_not_exist(self): - with self.assertRaises(RuntimeError): - interpreters.is_running(1_000_000) - - def test_bad_id(self): - with self.assertRaises(RuntimeError): - interpreters.is_running(-1) - - -class InterpreterIDTests(TestBase): - - def test_with_int(self): - id = interpreters.InterpreterID(10, force=True) - - self.assertEqual(int(id), 10) - - def test_coerce_id(self): - id = interpreters.InterpreterID('10', force=True) - self.assertEqual(int(id), 10) - - id = interpreters.InterpreterID(10.0, force=True) - self.assertEqual(int(id), 10) - - class Int(str): - def __init__(self, value): - self._value = value - def __int__(self): - return self._value - - id = interpreters.InterpreterID(Int(10), force=True) - self.assertEqual(int(id), 10) - - def test_bad_id(self): - for id in [-1, 'spam']: - with self.subTest(id): - with self.assertRaises(ValueError): - interpreters.InterpreterID(id) - with self.assertRaises(OverflowError): - interpreters.InterpreterID(2**64) - with self.assertRaises(TypeError): - interpreters.InterpreterID(object()) - - def test_does_not_exist(self): - id = interpreters.channel_create() - with self.assertRaises(RuntimeError): - interpreters.InterpreterID(int(id) + 1) # unforced - - def test_str(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = interpreters.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = interpreters.create() - id2 = interpreters.InterpreterID(int(id1)) - id3 = interpreters.create() - - self.assertTrue(id1 == id1) - self.assertTrue(id1 == id2) - self.assertTrue(id1 == int(id1)) - self.assertFalse(id1 == id3) - - self.assertFalse(id1 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id1 != id3) class CreateTests(TestBase): - def test_in_main(self): + def test_create(self): id = interpreters.create() - self.assertIsInstance(id, interpreters.InterpreterID) - self.assertIn(id, interpreters.list_all()) - @unittest.skip('enable this test when working on pystate.c') - def test_unique_id(self): - seen = set() - for _ in range(100): - id = interpreters.create() - interpreters.destroy(id) - seen.add(id) - - self.assertEqual(len(seen), 100) - - def test_in_thread(self): - lock = threading.Lock() - id = None - def f(): - nonlocal id - id = interpreters.create() - lock.acquire() - lock.release() - - t = threading.Thread(target=f) - with lock: - t.start() - t.join() - self.assertIn(id, interpreters.list_all()) - - def test_in_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() - out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters - id = _interpreters.create() - print(id) - assert isinstance(id, _interpreters.InterpreterID) - """)) - id2 = int(out.strip()) - - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) - - def test_in_threaded_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = None - def f(): - nonlocal id2 - out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters - id = _interpreters.create() - print(id) - """)) - id2 = int(out.strip()) - - t = threading.Thread(target=f) - t.start() - t.join() - - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) - - def test_after_destroy_all(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - ids = [] - for _ in range(3): - id = interpreters.create() - ids.append(id) - # Now destroy them. - for id in ids: - interpreters.destroy(id) - # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id}) - - def test_after_destroy_some(self): - before = set(interpreters.list_all()) - # Create 3 subinterpreters. - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() - # Now destroy 2 of them. - interpreters.destroy(id1) - interpreters.destroy(id3) - # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id, id2}) - - class DestroyTests(TestBase): - def test_one(self): + def test_destroy(self): id1 = interpreters.create() id2 = interpreters.create() id3 = interpreters.create() self.assertIn(id2, interpreters.list_all()) interpreters.destroy(id2) self.assertNotIn(id2, interpreters.list_all()) - self.assertIn(id1, interpreters.list_all()) - self.assertIn(id3, interpreters.list_all()) - - def test_all(self): - before = set(interpreters.list_all()) - ids = set() - for _ in range(3): - id = interpreters.create() - ids.add(id) - self.assertEqual(set(interpreters.list_all()), before | ids) - for id in ids: - interpreters.destroy(id) - self.assertEqual(set(interpreters.list_all()), before) - - def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - interpreters.destroy(main) - - def f(): - with self.assertRaises(RuntimeError): - interpreters.destroy(main) - - t = threading.Thread(target=f) - t.start() - t.join() - - def test_already_destroyed(self): - id = interpreters.create() - interpreters.destroy(id) - with self.assertRaises(RuntimeError): - interpreters.destroy(id) - - def test_does_not_exist(self): - with self.assertRaises(RuntimeError): - interpreters.destroy(1_000_000) - - def test_bad_id(self): - with self.assertRaises(RuntimeError): - interpreters.destroy(-1) - - def test_from_current(self): - main, = interpreters.list_all() - id = interpreters.create() - script = dedent(f""" - import _xxsubinterpreters as _interpreters - try: - _interpreters.destroy({id}) - except RuntimeError: - pass - """) - - interpreters.run_string(id, script) - self.assertEqual(set(interpreters.list_all()), {main, id}) - - def test_from_sibling(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = interpreters.create() - script = dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.destroy({id2}) - """) - interpreters.run_string(id1, script) - - self.assertEqual(set(interpreters.list_all()), {main, id1}) - - def test_from_other_thread(self): - id = interpreters.create() - def f(): - interpreters.destroy(id) - - t = threading.Thread(target=f) - t.start() - t.join() - - def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() - with _running(interp): - with self.assertRaises(RuntimeError): - interpreters.destroy(interp) - self.assertTrue(interpreters.is_running(interp)) class RunStringTests(TestBase): - SCRIPT = dedent(""" - with open('{}', 'w') as out: - out.write('{}') - """) - FILENAME = 'spam' - - def setUp(self): - super().setUp() - self.id = interpreters.create() - self._fs = None - - def tearDown(self): - if self._fs is not None: - self._fs.close() - super().tearDown() - - @property - def fs(self): - if self._fs is None: - self._fs = FSFixture(self) - return self._fs - - def test_success(self): - script, file = _captured_script('print("it worked!", end="")') - with file: - interpreters.run_string(self.id, script) - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_in_thread(self): + def test_run_string(self): script, file = _captured_script('print("it worked!", end="")') + id = interpreters.create() with file: - def f(): - interpreters.run_string(self.id, script) - - t = threading.Thread(target=f) - t.start() - t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_create_thread(self): - script, file = _captured_script(""" - import threading - def f(): - print('it worked!', end='') - - t = threading.Thread(target=f) - t.start() - t.join() - """) - with file: - interpreters.run_string(self.id, script) + interpreters.run_string(id, script, None) out = file.read() self.assertEqual(out, 'it worked!') - @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") - def test_fork(self): - import tempfile - with tempfile.NamedTemporaryFile('w+') as file: - file.write('') - file.flush() - - expected = 'spam spam spam spam spam' - script = dedent(f""" - import os - try: - os.fork() - except RuntimeError: - with open('{file.name}', 'w') as out: - out.write('{expected}') - """) - interpreters.run_string(self.id, script) - - file.seek(0) - content = file.read() - self.assertEqual(content, expected) - - def test_already_running(self): - with _running(self.id): - with self.assertRaises(RuntimeError): - interpreters.run_string(self.id, 'print("spam")') - - def test_does_not_exist(self): - id = 0 - while id in interpreters.list_all(): - id += 1 - with self.assertRaises(RuntimeError): - interpreters.run_string(id, 'print("spam")') - - def test_error_id(self): - with self.assertRaises(RuntimeError): - interpreters.run_string(-1, 'print("spam")') - - def test_bad_id(self): - with self.assertRaises(TypeError): - interpreters.run_string('spam', 'print("spam")') - - def test_bad_script(self): - with self.assertRaises(TypeError): - interpreters.run_string(self.id, 10) - - def test_bytes_for_script(self): - with self.assertRaises(TypeError): - interpreters.run_string(self.id, b'print("spam")') - - @contextlib.contextmanager - def assert_run_failed(self, exctype, msg=None): - with self.assertRaises(interpreters.RunFailedError) as caught: - yield - if msg is None: - self.assertEqual(str(caught.exception).split(':')[0], - str(exctype)) - else: - self.assertEqual(str(caught.exception), - "{}: {}".format(exctype, msg)) - - def test_invalid_syntax(self): - with self.assert_run_failed(SyntaxError): - # missing close paren - interpreters.run_string(self.id, 'print("spam"') - - def test_failure(self): - with self.assert_run_failed(Exception, 'spam'): - interpreters.run_string(self.id, 'raise Exception("spam")') - - def test_SystemExit(self): - with self.assert_run_failed(SystemExit, '42'): - interpreters.run_string(self.id, 'raise SystemExit(42)') - - def test_sys_exit(self): - with self.assert_run_failed(SystemExit): - interpreters.run_string(self.id, dedent(""" - import sys - sys.exit() - """)) - - with self.assert_run_failed(SystemExit, '42'): - interpreters.run_string(self.id, dedent(""" - import sys - sys.exit(42) - """)) - - def test_with_shared(self): - r, w = os.pipe() - - shared = { - 'spam': b'ham', - 'eggs': b'-1', - 'cheddar': None, - } - script = dedent(f""" - eggs = int(eggs) - spam = 42 - result = spam + eggs - - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """) - interpreters.run_string(self.id, script, shared) - with open(r, 'rb') as chan: - ns = pickle.load(chan) - - self.assertEqual(ns['spam'], 42) - self.assertEqual(ns['eggs'], -1) - self.assertEqual(ns['result'], 41) - self.assertIsNone(ns['cheddar']) - - def test_shared_overwrites(self): - interpreters.run_string(self.id, dedent(""" - spam = 'eggs' - ns1 = dict(vars()) - del ns1['__builtins__'] - """)) - - shared = {'spam': b'ham'} - script = dedent(f""" - ns2 = dict(vars()) - del ns2['__builtins__'] - """) - interpreters.run_string(self.id, script, shared) - - r, w = os.pipe() - script = dedent(f""" - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """) - interpreters.run_string(self.id, script) - with open(r, 'rb') as chan: - ns = pickle.load(chan) - - self.assertEqual(ns['ns1']['spam'], 'eggs') - self.assertEqual(ns['ns2']['spam'], b'ham') - self.assertEqual(ns['spam'], b'ham') - - def test_shared_overwrites_default_vars(self): - r, w = os.pipe() - - shared = {'__name__': b'not __main__'} - script = dedent(f""" - spam = 42 - - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """) - interpreters.run_string(self.id, script, shared) - with open(r, 'rb') as chan: - ns = pickle.load(chan) - - self.assertEqual(ns['__name__'], b'not __main__') - - def test_main_reused(self): - r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" - spam = True - - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - del ns, pickle, chan - """)) - with open(r, 'rb') as chan: - ns1 = pickle.load(chan) - - r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" - eggs = False - - ns = dict(vars()) - del ns['__builtins__'] - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """)) - with open(r, 'rb') as chan: - ns2 = pickle.load(chan) - - self.assertIn('spam', ns1) - self.assertNotIn('eggs', ns1) - self.assertIn('eggs', ns2) - self.assertIn('spam', ns2) - - def test_execution_namespace_is_main(self): - r, w = os.pipe() - - script = dedent(f""" - spam = 42 - - ns = dict(vars()) - ns['__builtins__'] = str(ns['__builtins__']) - import pickle - with open({w}, 'wb') as chan: - pickle.dump(ns, chan) - """) - interpreters.run_string(self.id, script) - with open(r, 'rb') as chan: - ns = pickle.load(chan) - - ns.pop('__builtins__') - ns.pop('__loader__') - self.assertEqual(ns, { - '__name__': '__main__', - '__annotations__': {}, - '__doc__': None, - '__package__': None, - '__spec__': None, - 'spam': 42, - }) - - # XXX Fix this test! - @unittest.skip('blocking forever') - def test_still_running_at_exit(self): - script = dedent(f""" - from textwrap import dedent - import threading - import _xxsubinterpreters as _interpreters - id = _interpreters.create() - def f(): - _interpreters.run_string(id, dedent(''' - import time - # Give plenty of time for the main interpreter to finish. - time.sleep(1_000_000) - ''')) - - t = threading.Thread(target=f) - t.start() - """) - with support.temp_dir() as dirname: - filename = script_helper.make_script(dirname, 'interp', script) - with script_helper.spawn_python(filename) as proc: - retcode = proc.wait() - - self.assertEqual(retcode, 0) - - -################################## -# channel tests - -class ChannelIDTests(TestBase): - - def test_default_kwargs(self): - cid = interpreters._channel_id(10, force=True) - - self.assertEqual(int(cid), 10) - self.assertEqual(cid.end, 'both') - - def test_with_kwargs(self): - cid = interpreters._channel_id(10, send=True, force=True) - self.assertEqual(cid.end, 'send') - - cid = interpreters._channel_id(10, send=True, recv=False, force=True) - self.assertEqual(cid.end, 'send') - - cid = interpreters._channel_id(10, recv=True, force=True) - self.assertEqual(cid.end, 'recv') - - cid = interpreters._channel_id(10, recv=True, send=False, force=True) - self.assertEqual(cid.end, 'recv') - - cid = interpreters._channel_id(10, send=True, recv=True, force=True) - self.assertEqual(cid.end, 'both') - - def test_coerce_id(self): - cid = interpreters._channel_id('10', force=True) - self.assertEqual(int(cid), 10) - - cid = interpreters._channel_id(10.0, force=True) - self.assertEqual(int(cid), 10) - - class Int(str): - def __init__(self, value): - self._value = value - def __int__(self): - return self._value - - cid = interpreters._channel_id(Int(10), force=True) - self.assertEqual(int(cid), 10) - - def test_bad_id(self): - for cid in [-1, 'spam']: - with self.subTest(cid): - with self.assertRaises(ValueError): - interpreters._channel_id(cid) - with self.assertRaises(OverflowError): - interpreters._channel_id(2**64) - with self.assertRaises(TypeError): - interpreters._channel_id(object()) - - def test_bad_kwargs(self): - with self.assertRaises(ValueError): - interpreters._channel_id(10, send=False, recv=False) - - def test_does_not_exist(self): - cid = interpreters.channel_create() - with self.assertRaises(interpreters.ChannelNotFoundError): - interpreters._channel_id(int(cid) + 1) # unforced - - def test_str(self): - cid = interpreters._channel_id(10, force=True) - self.assertEqual(str(cid), '10') - - def test_repr(self): - cid = interpreters._channel_id(10, force=True) - self.assertEqual(repr(cid), 'ChannelID(10)') - - cid = interpreters._channel_id(10, send=True, force=True) - self.assertEqual(repr(cid), 'ChannelID(10, send=True)') - - cid = interpreters._channel_id(10, recv=True, force=True) - self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') - - cid = interpreters._channel_id(10, send=True, recv=True, force=True) - self.assertEqual(repr(cid), 'ChannelID(10)') - - def test_equality(self): - cid1 = interpreters.channel_create() - cid2 = interpreters._channel_id(int(cid1)) - cid3 = interpreters.channel_create() - - self.assertTrue(cid1 == cid1) - self.assertTrue(cid1 == cid2) - self.assertTrue(cid1 == int(cid1)) - self.assertFalse(cid1 == cid3) - - self.assertFalse(cid1 != cid1) - self.assertFalse(cid1 != cid2) - self.assertTrue(cid1 != cid3) - - -class ChannelTests(TestBase): - - def test_create_cid(self): - cid = interpreters.channel_create() - self.assertIsInstance(cid, interpreters.ChannelID) - - def test_sequential_ids(self): - before = interpreters.channel_list_all() - id1 = interpreters.channel_create() - id2 = interpreters.channel_create() - id3 = interpreters.channel_create() - after = interpreters.channel_list_all() - - self.assertEqual(id2, int(id1) + 1) - self.assertEqual(id3, int(id2) + 1) - self.assertEqual(set(after) - set(before), {id1, id2, id3}) - - def test_ids_global(self): - id1 = interpreters.create() - out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters - cid = _interpreters.channel_create() - print(cid) - """)) - cid1 = int(out.strip()) - - id2 = interpreters.create() - out = _run_output(id2, dedent(""" - import _xxsubinterpreters as _interpreters - cid = _interpreters.channel_create() - print(cid) - """)) - cid2 = int(out.strip()) - - self.assertEqual(cid2, int(cid1) + 1) - - #################### - - def test_send_recv_main(self): - cid = interpreters.channel_create() - orig = b'spam' - interpreters.channel_send(cid, orig) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, orig) - self.assertIsNot(obj, orig) - - def test_send_recv_same_interpreter(self): - id1 = interpreters.create() - out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters - cid = _interpreters.channel_create() - orig = b'spam' - _interpreters.channel_send(cid, orig) - obj = _interpreters.channel_recv(cid) - assert obj is not orig - assert obj == orig - """)) - - def test_send_recv_different_interpreters(self): - cid = interpreters.channel_create() - id1 = interpreters.create() - out = _run_output(id1, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_send({cid}, b'spam') - """)) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - - def test_send_recv_different_threads(self): - cid = interpreters.channel_create() - - def f(): - while True: - try: - obj = interpreters.channel_recv(cid) - break - except interpreters.ChannelEmptyError: - time.sleep(0.1) - interpreters.channel_send(cid, obj) - t = threading.Thread(target=f) - t.start() - - interpreters.channel_send(cid, b'spam') - t.join() - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - - def test_send_recv_different_interpreters_and_threads(self): - cid = interpreters.channel_create() - id1 = interpreters.create() - out = None - - def f(): - nonlocal out - out = _run_output(id1, dedent(f""" - import time - import _xxsubinterpreters as _interpreters - while True: - try: - obj = _interpreters.channel_recv({cid}) - break - except _interpreters.ChannelEmptyError: - time.sleep(0.1) - assert(obj == b'spam') - _interpreters.channel_send({cid}, b'eggs') - """)) - t = threading.Thread(target=f) - t.start() - - interpreters.channel_send(cid, b'spam') - t.join() - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'eggs') - - def test_send_not_found(self): - with self.assertRaises(interpreters.ChannelNotFoundError): - interpreters.channel_send(10, b'spam') - - def test_recv_not_found(self): - with self.assertRaises(interpreters.ChannelNotFoundError): - interpreters.channel_recv(10) - - def test_recv_empty(self): - cid = interpreters.channel_create() - with self.assertRaises(interpreters.ChannelEmptyError): - interpreters.channel_recv(cid) - - def test_run_string_arg_unresolved(self): - cid = interpreters.channel_create() - interp = interpreters.create() - - out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - print(cid.end) - _interpreters.channel_send(cid, b'spam') - """), - dict(cid=cid.send)) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - self.assertEqual(out.strip(), 'send') - - # XXX For now there is no high-level channel into which the - # sent channel ID can be converted... - # Note: this test caused crashes on some buildbots (bpo-33615). - @unittest.skip('disabled until high-level channels exist') - def test_run_string_arg_resolved(self): - cid = interpreters.channel_create() - cid = interpreters._channel_id(cid, _resolve=True) - interp = interpreters.create() - - out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - print(chan.id.end) - _interpreters.channel_send(chan.id, b'spam') - """), - dict(chan=cid.send)) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - self.assertEqual(out.strip(), 'send') - - # close - - def test_close_single_user(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_close(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_multiple_users(self): - cid = interpreters.channel_create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_send({cid}, b'spam') - """)) - interpreters.run_string(id2, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_recv({cid}) - """)) - interpreters.channel_close(cid) - with self.assertRaises(interpreters.RunFailedError) as cm: - interpreters.run_string(id1, dedent(f""" - _interpreters.channel_send({cid}, b'spam') - """)) - self.assertIn('ChannelClosedError', str(cm.exception)) - with self.assertRaises(interpreters.RunFailedError) as cm: - interpreters.run_string(id2, dedent(f""" - _interpreters.channel_send({cid}, b'spam') - """)) - self.assertIn('ChannelClosedError', str(cm.exception)) - - def test_close_multiple_times(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_close(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(cid) - - def test_close_empty(self): - tests = [ - (False, False), - (True, False), - (False, True), - (True, True), - ] - for send, recv in tests: - with self.subTest((send, recv)): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_close(cid, send=send, recv=recv) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_defaults_with_unused_items(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - - with self.assertRaises(interpreters.ChannelNotEmptyError): - interpreters.channel_close(cid) - interpreters.channel_recv(cid) - interpreters.channel_send(cid, b'eggs') - - def test_close_recv_with_unused_items_unforced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - - with self.assertRaises(interpreters.ChannelNotEmptyError): - interpreters.channel_close(cid, recv=True) - interpreters.channel_recv(cid) - interpreters.channel_send(cid, b'eggs') - interpreters.channel_recv(cid) - interpreters.channel_recv(cid) - interpreters.channel_close(cid, recv=True) - - def test_close_send_with_unused_items_unforced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, send=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - interpreters.channel_recv(cid) - interpreters.channel_recv(cid) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_both_with_unused_items_unforced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - - with self.assertRaises(interpreters.ChannelNotEmptyError): - interpreters.channel_close(cid, recv=True, send=True) - interpreters.channel_recv(cid) - interpreters.channel_send(cid, b'eggs') - interpreters.channel_recv(cid) - interpreters.channel_recv(cid) - interpreters.channel_close(cid, recv=True) - - def test_close_recv_with_unused_items_forced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, recv=True, force=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_send_with_unused_items_forced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, send=True, force=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_both_with_unused_items_forced(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, send=True, recv=True, force=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_never_used(self): - cid = interpreters.channel_create() - interpreters.channel_close(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'spam') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close_by_unassociated_interp(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_close({cid}, force=True) - """)) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(cid) - - def test_close_used_multiple_times_by_single_user(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_close(cid, force=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - -class ChannelReleaseTests(TestBase): - - # XXX Add more test coverage a la the tests for close(). - - """ - - main / interp / other - - run in: current thread / new thread / other thread / different threads - - end / opposite - - force / no force - - used / not used (associated / not associated) - - empty / emptied / never emptied / partly emptied - - closed / not closed - - released / not released - - creator (interp) / other - - associated interpreter not running - - associated interpreter destroyed - """ - - """ - use - pre-release - release - after - check - """ - - """ - release in: main, interp1 - creator: same, other (incl. interp2) - - use: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all - pre-release: None,send,recv,both in None,same,other(incl. interp2),same+other(incl. interp2),all - pre-release forced: None,send,recv,both in None,same,other(incl. interp2),same+other(incl. interp2),all - - release: same - release forced: same - - use after: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all - release after: None,send,recv,send/recv in None,same,other(incl. interp2),same+other(incl. interp2),all - check released: send/recv for same/other(incl. interp2) - check closed: send/recv for same/other(incl. interp2) - """ - - def test_single_user(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_release(cid, send=True, recv=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_multiple_users(self): - cid = interpreters.channel_create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_send({cid}, b'spam') - """)) - out = _run_output(id2, dedent(f""" - import _xxsubinterpreters as _interpreters - obj = _interpreters.channel_recv({cid}) - _interpreters.channel_release({cid}) - print(repr(obj)) - """)) - interpreters.run_string(id1, dedent(f""" - _interpreters.channel_release({cid}) - """)) - - self.assertEqual(out.strip(), "b'spam'") - - def test_no_kwargs(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_multiple_times(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_release(cid, send=True, recv=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_release(cid, send=True, recv=True) - - def test_with_unused_items(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'ham') - interpreters.channel_release(cid, send=True, recv=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_never_used(self): - cid = interpreters.channel_create() - interpreters.channel_release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'spam') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_by_unassociated_interp(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - _interpreters.channel_release({cid}) - """)) - obj = interpreters.channel_recv(cid) - interpreters.channel_release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - self.assertEqual(obj, b'spam') - - def test_close_if_unassociated(self): - # XXX Something's not right with this test... - cid = interpreters.channel_create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxsubinterpreters as _interpreters - obj = _interpreters.channel_send({cid}, b'spam') - _interpreters.channel_release({cid}) - """)) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_partially(self): - # XXX Is partial close too weird/confusing? - cid = interpreters.channel_create() - interpreters.channel_send(cid, None) - interpreters.channel_recv(cid) - interpreters.channel_send(cid, b'spam') - interpreters.channel_release(cid, send=True) - obj = interpreters.channel_recv(cid) - - self.assertEqual(obj, b'spam') - - def test_used_multiple_times_by_single_user(self): - cid = interpreters.channel_create() - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'spam') - interpreters.channel_send(cid, b'spam') - interpreters.channel_recv(cid) - interpreters.channel_release(cid, send=True, recv=True) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - -class ChannelCloseFixture(namedtuple('ChannelCloseFixture', - 'end interp other extra creator')): - - # Set this to True to avoid creating interpreters, e.g. when - # scanning through test permutations without running them. - QUICK = False - - def __new__(cls, end, interp, other, extra, creator): - assert end in ('send', 'recv') - if cls.QUICK: - known = {} - else: - interp = Interpreter.from_raw(interp) - other = Interpreter.from_raw(other) - extra = Interpreter.from_raw(extra) - known = { - interp.name: interp, - other.name: other, - extra.name: extra, - } - if not creator: - creator = 'same' - self = super().__new__(cls, end, interp, other, extra, creator) - self._prepped = set() - self._state = ChannelState() - self._known = known - return self - - @property - def state(self): - return self._state - - @property - def cid(self): - try: - return self._cid - except AttributeError: - creator = self._get_interpreter(self.creator) - self._cid = self._new_channel(creator) - return self._cid - - def get_interpreter(self, interp): - interp = self._get_interpreter(interp) - self._prep_interpreter(interp) - return interp - - def expect_closed_error(self, end=None): - if end is None: - end = self.end - if end == 'recv' and self.state.closed == 'send': - return False - return bool(self.state.closed) - - def prep_interpreter(self, interp): - self._prep_interpreter(interp) - - def record_action(self, action, result): - self._state = result - - def clean_up(self): - clean_up_interpreters() - clean_up_channels() - - # internal methods - - def _new_channel(self, creator): - if creator.name == 'main': - return interpreters.channel_create() - else: - ch = interpreters.channel_create() - run_interp(creator.id, f""" - import _xxsubinterpreters - cid = _xxsubinterpreters.channel_create() - # We purposefully send back an int to avoid tying the - # channel to the other interpreter. - _xxsubinterpreters.channel_send({ch}, int(cid)) - del _xxsubinterpreters - """) - self._cid = interpreters.channel_recv(ch) - return self._cid - - def _get_interpreter(self, interp): - if interp in ('same', 'interp'): - return self.interp - elif interp == 'other': - return self.other - elif interp == 'extra': - return self.extra - else: - name = interp - try: - interp = self._known[name] - except KeyError: - interp = self._known[name] = Interpreter(name) - return interp - - def _prep_interpreter(self, interp): - if interp.id in self._prepped: - return - self._prepped.add(interp.id) - if interp.name == 'main': - return - run_interp(interp.id, f""" - import _xxsubinterpreters as interpreters - import test.test__xxsubinterpreters as helpers - ChannelState = helpers.ChannelState - try: - cid - except NameError: - cid = interpreters._channel_id({self.cid}) - """) - - -@unittest.skip('these tests take several hours to run') -class ExhaustiveChannelTests(TestBase): - - """ - - main / interp / other - - run in: current thread / new thread / other thread / different threads - - end / opposite - - force / no force - - used / not used (associated / not associated) - - empty / emptied / never emptied / partly emptied - - closed / not closed - - released / not released - - creator (interp) / other - - associated interpreter not running - - associated interpreter destroyed - - - close after unbound - """ - - """ - use - pre-close - close - after - check - """ - - """ - close in: main, interp1 - creator: same, other, extra - - use: None,send,recv,send/recv in None,same,other,same+other,all - pre-close: None,send,recv in None,same,other,same+other,all - pre-close forced: None,send,recv in None,same,other,same+other,all - - close: same - close forced: same - - use after: None,send,recv,send/recv in None,same,other,extra,same+other,all - close after: None,send,recv,send/recv in None,same,other,extra,same+other,all - check closed: send/recv for same/other(incl. interp2) - """ - - def iter_action_sets(self): - # - used / not used (associated / not associated) - # - empty / emptied / never emptied / partly emptied - # - closed / not closed - # - released / not released - - # never used - yield [] - - # only pre-closed (and possible used after) - for closeactions in self._iter_close_action_sets('same', 'other'): - yield closeactions - for postactions in self._iter_post_close_action_sets(): - yield closeactions + postactions - for closeactions in self._iter_close_action_sets('other', 'extra'): - yield closeactions - for postactions in self._iter_post_close_action_sets(): - yield closeactions + postactions - - # used - for useactions in self._iter_use_action_sets('same', 'other'): - yield useactions - for closeactions in self._iter_close_action_sets('same', 'other'): - actions = useactions + closeactions - yield actions - for postactions in self._iter_post_close_action_sets(): - yield actions + postactions - for closeactions in self._iter_close_action_sets('other', 'extra'): - actions = useactions + closeactions - yield actions - for postactions in self._iter_post_close_action_sets(): - yield actions + postactions - for useactions in self._iter_use_action_sets('other', 'extra'): - yield useactions - for closeactions in self._iter_close_action_sets('same', 'other'): - actions = useactions + closeactions - yield actions - for postactions in self._iter_post_close_action_sets(): - yield actions + postactions - for closeactions in self._iter_close_action_sets('other', 'extra'): - actions = useactions + closeactions - yield actions - for postactions in self._iter_post_close_action_sets(): - yield actions + postactions - - def _iter_use_action_sets(self, interp1, interp2): - interps = (interp1, interp2) - - # only recv end used - yield [ - ChannelAction('use', 'recv', interp1), - ] - yield [ - ChannelAction('use', 'recv', interp2), - ] - yield [ - ChannelAction('use', 'recv', interp1), - ChannelAction('use', 'recv', interp2), - ] - - # never emptied - yield [ - ChannelAction('use', 'send', interp1), - ] - yield [ - ChannelAction('use', 'send', interp2), - ] - yield [ - ChannelAction('use', 'send', interp1), - ChannelAction('use', 'send', interp2), - ] - - # partially emptied - for interp1 in interps: - for interp2 in interps: - for interp3 in interps: - yield [ - ChannelAction('use', 'send', interp1), - ChannelAction('use', 'send', interp2), - ChannelAction('use', 'recv', interp3), - ] - - # fully emptied - for interp1 in interps: - for interp2 in interps: - for interp3 in interps: - for interp4 in interps: - yield [ - ChannelAction('use', 'send', interp1), - ChannelAction('use', 'send', interp2), - ChannelAction('use', 'recv', interp3), - ChannelAction('use', 'recv', interp4), - ] - - def _iter_close_action_sets(self, interp1, interp2): - ends = ('recv', 'send') - interps = (interp1, interp2) - for force in (True, False): - op = 'force-close' if force else 'close' - for interp in interps: - for end in ends: - yield [ - ChannelAction(op, end, interp), - ] - for recvop in ('close', 'force-close'): - for sendop in ('close', 'force-close'): - for recv in interps: - for send in interps: - yield [ - ChannelAction(recvop, 'recv', recv), - ChannelAction(sendop, 'send', send), - ] - - def _iter_post_close_action_sets(self): - for interp in ('same', 'extra', 'other'): - yield [ - ChannelAction('use', 'recv', interp), - ] - yield [ - ChannelAction('use', 'send', interp), - ] - - def run_actions(self, fix, actions): - for action in actions: - self.run_action(fix, action) - - def run_action(self, fix, action, *, hideclosed=True): - end = action.resolve_end(fix.end) - interp = action.resolve_interp(fix.interp, fix.other, fix.extra) - fix.prep_interpreter(interp) - if interp.name == 'main': - result = run_action( - fix.cid, - action.action, - end, - fix.state, - hideclosed=hideclosed, - ) - fix.record_action(action, result) - else: - _cid = interpreters.channel_create() - run_interp(interp.id, f""" - result = helpers.run_action( - {fix.cid}, - {repr(action.action)}, - {repr(end)}, - {repr(fix.state)}, - hideclosed={hideclosed}, - ) - interpreters.channel_send({_cid}, result.pending.to_bytes(1, 'little')) - interpreters.channel_send({_cid}, b'X' if result.closed else b'') - """) - result = ChannelState( - pending=int.from_bytes(interpreters.channel_recv(_cid), 'little'), - closed=bool(interpreters.channel_recv(_cid)), - ) - fix.record_action(action, result) - - def iter_fixtures(self): - # XXX threads? - interpreters = [ - ('main', 'interp', 'extra'), - ('interp', 'main', 'extra'), - ('interp1', 'interp2', 'extra'), - ('interp1', 'interp2', 'main'), - ] - for interp, other, extra in interpreters: - for creator in ('same', 'other', 'creator'): - for end in ('send', 'recv'): - yield ChannelCloseFixture(end, interp, other, extra, creator) - - def _close(self, fix, *, force): - op = 'force-close' if force else 'close' - close = ChannelAction(op, fix.end, 'same') - if not fix.expect_closed_error(): - self.run_action(fix, close, hideclosed=False) - else: - with self.assertRaises(interpreters.ChannelClosedError): - self.run_action(fix, close, hideclosed=False) - - def _assert_closed_in_interp(self, fix, interp=None): - if interp is None or interp.name == 'main': - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(fix.cid) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(fix.cid, b'spam') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(fix.cid) - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(fix.cid, force=True) - else: - run_interp(interp.id, f""" - with helpers.expect_channel_closed(): - interpreters.channel_recv(cid) - """) - run_interp(interp.id, f""" - with helpers.expect_channel_closed(): - interpreters.channel_send(cid, b'spam') - """) - run_interp(interp.id, f""" - with helpers.expect_channel_closed(): - interpreters.channel_close(cid) - """) - run_interp(interp.id, f""" - with helpers.expect_channel_closed(): - interpreters.channel_close(cid, force=True) - """) - - def _assert_closed(self, fix): - self.assertTrue(fix.state.closed) - - for _ in range(fix.state.pending): - interpreters.channel_recv(fix.cid) - self._assert_closed_in_interp(fix) - - for interp in ('same', 'other'): - interp = fix.get_interpreter(interp) - if interp.name == 'main': - continue - self._assert_closed_in_interp(fix, interp) - - interp = fix.get_interpreter('fresh') - self._assert_closed_in_interp(fix, interp) - - def _iter_close_tests(self, verbose=False): - i = 0 - for actions in self.iter_action_sets(): - print() - for fix in self.iter_fixtures(): - i += 1 - if i > 1000: - return - if verbose: - if (i - 1) % 6 == 0: - print() - print(i, fix, '({} actions)'.format(len(actions))) - else: - if (i - 1) % 6 == 0: - print(' ', end='') - print('.', end=''); sys.stdout.flush() - yield i, fix, actions - if verbose: - print('---') - print() - - # This is useful for scanning through the possible tests. - def _skim_close_tests(self): - ChannelCloseFixture.QUICK = True - for i, fix, actions in self._iter_close_tests(): - pass - - def test_close(self): - for i, fix, actions in self._iter_close_tests(): - with self.subTest('{} {} {}'.format(i, fix, actions)): - fix.prep_interpreter(fix.interp) - self.run_actions(fix, actions) - - self._close(fix, force=False) - - self._assert_closed(fix) - # XXX Things slow down if we have too many interpreters. - fix.clean_up() - - def test_force_close(self): - for i, fix, actions in self._iter_close_tests(): - with self.subTest('{} {} {}'.format(i, fix, actions)): - fix.prep_interpreter(fix.interp) - self.run_actions(fix, actions) - - self._close(fix, force=True) - - self._assert_closed(fix) - # XXX Things slow down if we have too many interpreters. - fix.clean_up() - if __name__ == '__main__': unittest.main() From 4618343e07ea2610ee2c8003521b15aeaf530d27 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 10 Sep 2019 16:50:27 +0000 Subject: [PATCH 04/20] update __all__ --- Doc/library/interpreters.rst | 3 ++- Lib/interpreters.py | 9 ++++----- Lib/test/test_interpreters.py | 12 +----------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 28ab95c370ce66..c2f8d19e495878 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -31,7 +31,8 @@ This module defines the following functions: .. function:: destroy(id) - Destroy the interpreter whose ID is *id*. + Destroy the interpreter whose ID is *id*. Attempting to destroy the current + interpreter results in a `RuntimeError`. So does an unrecognized ID. .. function:: get_main() diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 95537bba6cbdae..62ca1466faa686 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -1,8 +1,10 @@ """Sub-interpreters High Level Module.""" -import _interpreters +__all__ = ['create', 'list_all', 'get_current', 'get_main', + 'run_string', 'destroy'] + -__all__ = ['create', 'list_all', 'get_current'] +import _interpreters # Rename so that "from interpreters import *" is safe _list_all = _interpreters.list_all @@ -38,7 +40,6 @@ def get_main(): Return the ID of the main interpreter. """ - return _get_main() def destroy(id): @@ -46,7 +47,6 @@ def destroy(id): Destroy the identified interpreter. """ - return _interpreters.destroy(id) def run_string(id, script, shared): @@ -55,5 +55,4 @@ def run_string(id, script, shared): Execute the provided string in the identified interpreter. See PyRun_SimpleStrings. """ - return _run_string(id, script, shared) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 32964ec435f4c0..d938235989720f 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,19 +1,8 @@ import interpreters -import _interpreters #remove when all methods are implemented -from collections import namedtuple -import contextlib -import itertools import os -import pickle -import sys from textwrap import dedent -import threading -import time import unittest -from test import support -from test.support import script_helper - def _captured_script(script): r, w = os.pipe() indented = script.replace('\n', '\n ') @@ -71,6 +60,7 @@ def test_create(self): id = interpreters.create() self.assertIn(id, interpreters.list_all()) + class DestroyTests(TestBase): def test_destroy(self): From ec63183c3b20fd4cd33b64bf37f7102be02648f2 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Thu, 12 Sep 2019 17:01:57 +0000 Subject: [PATCH 05/20] Implementation of all methods and relevant documentation --- Doc/library/interpreters.rst | 153 +++++++++++++++++++++-- Lib/interpreters.py | 234 ++++++++++++++++++++++++++++++----- 2 files changed, 340 insertions(+), 47 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index c2f8d19e495878..9df38111a65ab8 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -13,32 +13,159 @@ level :mod:`_interpreters` module. .. versionchanged:: 3.9 +Interpreter Objects +------------------- -This module defines the following functions: +The Interpreter object represents a single interpreter. +.. class:: Interpreter(id) + + The class implementing a subinterpreter object. + + .. method:: is_running() + + Return whether or not the identified interpreter is running. + + .. method:: destroy() + + Destroy the identified interpreter. Attempting to destroy the current + interpreter results in a RuntimeError. So does an unrecognized ID. + + .. method:: run(self, src_str, /, *, channels=None): + + Run the given source code in the interpreter. This blocks the current + thread until done. + +RecvChannel Objects +------------------- + +The RecvChannel object represents a recieving channel. + +.. class:: RecvChannel(id) + + This class represents the receiving end of a channel. + + .. method:: recv() + + Get the next object from the channel, and wait if none have been + sent. Associate the interpreter with the channel. + + .. method:: recv_nowait(default=None) + + Like ``recv()``, but return the default instead of waiting. + + .. method:: release() + + No longer associate the current interpreter with the channel + (on the sending end). + + .. method:: close(force=False) + + Close the channel in all interpreters. + + +SendChannel Objects +-------------------- + +The SendChannel object represents a sending channel. + +.. class:: SendChannel(id) + + This class represents the receiving end of a channel. + + .. method:: send(obj) + + Send the object (i.e. its data) to the receiving end of the channel + and wait.Associate the interpreter with the channel. + + .. method:: send_nowait(obj) + + Like ``send()``, but return False if not received. + + .. method:: send_buffer(obj) + + Send the object's buffer to the receiving end of the channel and wait. + Associate the interpreter with the channel. + + .. method:: send_buffer_nowait(obj) + + Like ``send_buffer()``, but return False if not received. + + .. method:: release() + + No longer associate the current interpreter with the channel + (on the sending end). + + .. method:: close(force=False) + + Close the channel in all interpreters. + + +This module defines the following global functions: + + +.. function:: is_shareable(obj) + + Return `True` if the object's data can be shared between interpreters. + +.. function:: create_channel() + + Create a new channel for passing data between interpreters. + +.. function:: list_all_channels() + + Return all open channels. .. function:: create() - Create a new interpreter and return a unique generated ID. + Initialize a new (idle) Python interpreter. + +.. function:: get_current() + + Get the currently running interpreter. .. function:: list_all() - Return a list containing the ID of every existing interpreter. + Get all existing interpreters. -.. function:: get_current() +This module also defines the following exceptions. + +.. exception:: RunFailedError + + This exception, a subclass of :exc:`RuntimeError`, is raised when the + ``Interpreter.run()`` results in an uncaught exception. + +.. exception:: ChannelError + + This exception, a subclass of :exc:`Exception`, and is the base class for + channel-related exceptions. + +.. exception:: ChannelNotFoundError + + This exception, a subclass of :exc:`ChannelError`, is raised when the + the identified channel was not found. + +.. exception:: ChannelEmptyError + + This exception, a subclass of :exc:`ChannelError`, is raised when + the channel is unexpectedly empty. + +.. exception:: ChannelNotEmptyError - Return the ID of the currently running interpreter. + This exception, a subclass of :exc:`ChannelError`, is raised when + the channel is unexpectedly not empty. -.. function:: destroy(id) +.. exception:: NotReceivedError - Destroy the interpreter whose ID is *id*. Attempting to destroy the current - interpreter results in a `RuntimeError`. So does an unrecognized ID. + This exception, a subclass of :exc:`ChannelError`, is raised when + nothing was waiting to receive a sent object. -.. function:: get_main() +.. exception:: ChannelClosedError - Return the ID of the main interpreter. + This exception, a subclass of :exc:`ChannelError`, is raised when + the channel is closed. -.. function:: run_string() +.. exception:: ChannelReleasedError - Execute the provided string in the identified interpreter. - See `PyRun_SimpleStrings`. + This exception, a subclass of :exc:`ChannelClosedError`, is raised when + the channel is released (but not yet closed). diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 62ca1466faa686..8bed804bb25b68 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -1,58 +1,224 @@ -"""Sub-interpreters High Level Module.""" +"""Subinterpreters High Level Module.""" -__all__ = ['create', 'list_all', 'get_current', 'get_main', - 'run_string', 'destroy'] +import _interpreters +import logger +__all__ = _all__ = ['create', 'list_all', 'get_current', 'get_main', + 'run_string', 'destroy'] -import _interpreters -# Rename so that "from interpreters import *" is safe -_list_all = _interpreters.list_all -_get_current = _interpreters.get_current -_get_main = _interpreters.get_main -_run_string = _interpreters.run_string +class Interpreter: + + def __init__(self, id): + self.id = id + + def is_running(self): + """is_running() -> bool + + Return whether or not the identified interpreter is running. + """ + return _interpreters.is_running(self.id) + + def destroy(self): + """destroy() + + Destroy the identified interpreter. + + Attempting to destroy the current + interpreter results in a RuntimeError. So does an unrecognized ID + """ + return _interpreters.destroy(self.id) + + def run(self, src_str, /, *, channels=None): + """run(src_str, /, *, channels=None) + + Run the given source code in the interpreter. + This blocks the current thread until done. + """ + try: + _interpreters.run_string(self.id, src_str) + except RunFailedError as err: + logger.error(err) + raise + +def wait(self, timeout): + #The implementation for wait + # will be non trivial to be useful + import time + time.sleep(timeout) + +def associate_interp_to_channel(id, cid): + pass + +class RecvChannel: + + def __init__(self, id): + self.id = id + self.interpreters = _interpreters.list_all() + + def recv(self, timeout=2): + """ channel_recv() -> obj + + + Get the next object from the channel, + and wait if none have been sent. + Associate the interpreter with the channel. + """ + obj = _interpreters.channel_recv(self.id) + if obj == None: + wait(timeout) + obj = obj = _interpreters.channel_recv(self.id) + + associate_interp_to_channel(interpId, Cid) + + return obj + + def recv_nowait(self, default=None): + """recv_nowait(default=None) -> object + + Like recv(), but return the default instead of waiting. + """ + return _interpreters.channel_recv(self.id) + + def release(self): + """ release() + + No longer associate the current interpreterwith the channel + (on the sending end). + """ + pass + + def close(self, force=False): + """close(force=False) + + Close the channel in all interpreters.. + """ + return _interpreters.channel_close(self.id, force) + + +class SendChannel: + + def __init__(self, id): + self.id = id + self.interpreters = _interpreters.list_all() + + def send(self, obj, timeout=2): + """ send(obj) + + Send the object (i.e. its data) to the receiving end of the channel + and wait. Associate the interpreter with the channel. + """ + obj = _interpreters.channel_send(self.id, obj) + wait(timeout) + associate_interp_to_channel(interpId, Cid) + + def send_nowait(self, obj): + """ send_nowait(obj) + + Like send(), but return False if not received. + """ + try: + obj = _interpreters.channel_send(self.id, obj) + except: + return False + + return obj + + def release(self): + """ release() + + No longer associate the current interpreterwith the channel + (on the sending end). + """ + pass + + def close(self, force=False): + """ close(force=False) + + No longer associate the current interpreterwith the channel + (on the sending end). + """ + return _interpreters.channel_close(self.id, force) + + +class ChannelError(Exception): + pass + + +class ChannelNotFoundError(ChannelError): + pass + + +class ChannelEmptyError(ChannelError): + pass + + +class ChannelNotEmptyError(ChannelError): + pass + + +class NotReceivedError(ChannelError): + pass + + +class ChannelClosedError(ChannelError): + pass + + +class ChannelReleasedError(ChannelClosedError): + pass + + +class RunFailedError(RuntimeError): + pass + # Global API functions -def create(): - """ create() -> Interpreter +def is_shareable(obj): + """ is_shareable(obj) -> Bool - Create a new interpreter and return a unique generated ID. + Return `True` if the object's data can be shared between + interpreters. """ - return _interpreters.create() + return _interpreters.is_shareable(obj) -def list_all(): - """list_all() -> [Interpreter] +def create_channel(): + """ create_channel() -> (RecvChannel, SendChannel) - Return a list containing the ID of every existing interpreter. + Create a new channel for passing data between interpreters. """ - return _list_all() -def get_current(): - """get_current() -> Interpreter + cid = _interpreters.channel_create() + return (RecvChannel(cid), SendChannel(cid)) - Return the ID of the currently running interpreter. +def list_all_channels(): + """ list_all_channels() -> [(RecvChannel, SendChannel)] + + Return all open channels. """ - return _get_current() + cid = _interpreters.channel_list_all() + return (RecvChannel(cid), SendChannel(cid)) -def get_main(): - """get_main() -> ID +def create(): + """ create() -> Interpreter - Return the ID of the main interpreter. + Initialize a new (idle) Python interpreter. """ - return _get_main() + id = _interpreters.create() + return Interpreter(id) -def destroy(id): - """destroy(id) -> None +def list_all(): + """ list_all() -> [Interpreter] - Destroy the identified interpreter. + Get all existing interpreters. """ - return _interpreters.destroy(id) + return [Interpreter(id) for id in _interpreters.list_all()] -def run_string(id, script, shared): - """run_string(id, script, shared) -> None +def get_current(): + """ get_current() -> Interpreter - Execute the provided string in the identified interpreter. - See PyRun_SimpleStrings. + Get the currently running interpreter. """ - return _run_string(id, script, shared) + id = _interpreters.get_current() + return Interpreter(id) From 80f680867593fd7562c642ed19881147006cdbba Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:30:29 -0300 Subject: [PATCH 06/20] Update Lib/test/test_interpreters.py Co-Authored-By: Eric Snow --- Lib/test/test_interpreters.py | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index d938235989720f..5f4dda331f0a5c 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -24,6 +24,44 @@ def clean_up_interpreters(): pass # already destroyed +class LowLevelStub: + # set these as appropriate in tests + errors = () + return_create = () + ... + def __init__(self): + self._calls = [] + def _add_call(self, name, args=(), kwargs=None): + self.calls.append( + (name, args, kwargs or {})) + def _maybe_error(self): + if not self.errors: + return + err = self.errors.pop(0) + if err is not None: + raise err + def check_calls(self, test, expected): + test.assertEqual(self._calls, expected) + for returns in [self.errors, self.return_create, ...]: + test.assertEqual(tuple(returns), ()) # make sure all were used + + # the stubbed methods + def create(self): + self._add_call('create') + self._maybe_error() + return self.return_create.pop(0) + def list_all(self): + ... + def get_current(self): + ... + def get_main(self): + ... + def destroy(self, id): + ... + def run_string(self, id, text, ...): + ... + + class TestBase(unittest.TestCase): def tearDown(self): From 724e618fc3e1e95c8e839c83a48ba1b2bc36e074 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:30:42 -0300 Subject: [PATCH 07/20] Update Lib/test/test_interpreters.py Co-Authored-By: Eric Snow --- Lib/test/test_interpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 5f4dda331f0a5c..93b06bf13745d3 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -96,7 +96,7 @@ class CreateTests(TestBase): def test_create(self): id = interpreters.create() - self.assertIn(id, interpreters.list_all()) + self.assertIn(interp, interpreters.list_all()) class DestroyTests(TestBase): From abd0011e000b2b7f1dbfe8ad3abf720d24a9c13e Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:30:57 -0300 Subject: [PATCH 08/20] Update Lib/test/test_interpreters.py Co-Authored-By: Eric Snow --- Lib/test/test_interpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 93b06bf13745d3..6044977d608caa 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -95,7 +95,7 @@ def test_get_main(self): class CreateTests(TestBase): def test_create(self): - id = interpreters.create() + interp = interpreters.create() self.assertIn(interp, interpreters.list_all()) From d502415deafb2fdc42e6234e2fa237f223444eb8 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:31:13 -0300 Subject: [PATCH 09/20] Update Lib/test/test_interpreters.py Co-Authored-By: Eric Snow --- Lib/test/test_interpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 6044977d608caa..2ca2d402d234d3 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -87,7 +87,7 @@ def test_get_current(self): class GetMainTests(TestBase): def test_get_main(self): - [expected] = interpreters.list_all() + expected, * = interpreters.list_all() main = interpreters.get_main() self.assertEqual(main, expected) From a751e153b2f81d5310d3d52a0e616dd708055f28 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:31:38 -0300 Subject: [PATCH 10/20] Update Doc/library/interpreters.rst Co-Authored-By: Eric Snow --- Doc/library/interpreters.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 9df38111a65ab8..e4938a69abef72 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -17,7 +17,6 @@ Interpreter Objects ------------------- The Interpreter object represents a single interpreter. - .. class:: Interpreter(id) The class implementing a subinterpreter object. From 56083c2079e73e639a3019e7b25b68fd5d819cb8 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:31:50 -0300 Subject: [PATCH 11/20] Update Doc/library/interpreters.rst Co-Authored-By: Eric Snow --- Doc/library/interpreters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index e4938a69abef72..ed935f16f0a315 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -11,7 +11,7 @@ This module constructs higher-level interpreters interfaces on top of the lower level :mod:`_interpreters` module. -.. versionchanged:: 3.9 +.. versionchanged:: added in 3.9 Interpreter Objects ------------------- From 14e97fcf29e26d347937cad1c594d0fe9f162202 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Fri, 13 Sep 2019 05:32:07 -0300 Subject: [PATCH 12/20] Update Doc/library/interpreters.rst Co-Authored-By: Eric Snow --- Doc/library/interpreters.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index ed935f16f0a315..8c4d44a43f2b42 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -8,7 +8,8 @@ -------------- -This module constructs higher-level interpreters interfaces on top of the lower +This module provides tools for working with sub-interpreters, such as creating them, +running code in them, or sending data between them. It is a wrapper around the low- level :mod:`_interpreters` module. .. versionchanged:: added in 3.9 From 9ab54eb47764f1508b23936375cc890f93c9c074 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Fri, 13 Sep 2019 10:52:53 +0000 Subject: [PATCH 13/20] Add tests --- Doc/library/interpreters.rst | 14 +-- Lib/interpreters.py | 32 +++++-- Lib/test/test_interpreters.py | 176 ++++++++++++++++++++-------------- 3 files changed, 138 insertions(+), 84 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 8c4d44a43f2b42..5d15484b6b4d78 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -1,8 +1,8 @@ -:mod:`interpreters` --- High-level Sub-interpreters Module +:mod:`interpreters` --- High-level Subinterpreters Module ========================================================== .. module:: interpreters - :synopsis: High-level Sub-Interpreters Module. + :synopsis: High-level SubInterpreters Module. **Source code:** :source:`Lib/interpreters.py` @@ -56,8 +56,9 @@ The RecvChannel object represents a recieving channel. .. method:: release() - No longer associate the current interpreter with the channel - (on the sending end). + Close the channel for the current interpreter. 'send' and 'recv' (bool) may + be used to indicate the ends to close. By default both ends are closed. + Closing an already closed end is a noop. .. method:: close(force=False) @@ -93,8 +94,9 @@ The SendChannel object represents a sending channel. .. method:: release() - No longer associate the current interpreter with the channel - (on the sending end). + Close the channel for the current interpreter. 'send' and 'recv' (bool) may + be used to indicate the ends to close. By default both ends are closed. + Closing an already closed end is a noop. .. method:: close(force=False) diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 8bed804bb25b68..9fa3877540408f 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -3,8 +3,9 @@ import _interpreters import logger -__all__ = _all__ = ['create', 'list_all', 'get_current', 'get_main', - 'run_string', 'destroy'] +__all__ = ['Interpreter', 'SendChannel', 'RecvChannel', 'is_shareable', + 'create_channel', 'list_all_channels', 'list_all', 'get_current', + 'create'] class Interpreter: @@ -59,7 +60,6 @@ def __init__(self, id): def recv(self, timeout=2): """ channel_recv() -> obj - Get the next object from the channel, and wait if none have been sent. Associate the interpreter with the channel. @@ -69,6 +69,7 @@ def recv(self, timeout=2): wait(timeout) obj = obj = _interpreters.channel_recv(self.id) + # Pending: See issue 52 on multi-core python project associate_interp_to_channel(interpId, Cid) return obj @@ -80,13 +81,28 @@ def recv_nowait(self, default=None): """ return _interpreters.channel_recv(self.id) + def send_buffer(self, obj): + """ send_buffer(obj) + + Send the object's buffer to the receiving end of the channel + and wait. Associate the interpreter with the channel. + """ + pass + + def send_buffer_nowait(self, obj): + """ send_buffer_nowait(obj) + + Like send_buffer(), but return False if not received. + """ + pass + def release(self): """ release() No longer associate the current interpreterwith the channel (on the sending end). """ - pass + return _interpreters.(self.id) def close(self, force=False): """close(force=False) @@ -102,14 +118,14 @@ def __init__(self, id): self.id = id self.interpreters = _interpreters.list_all() - def send(self, obj, timeout=2): + def send(self, obj): """ send(obj) Send the object (i.e. its data) to the receiving end of the channel and wait. Associate the interpreter with the channel. """ obj = _interpreters.channel_send(self.id, obj) - wait(timeout) + wait(2) associate_interp_to_channel(interpId, Cid) def send_nowait(self, obj): @@ -127,10 +143,10 @@ def send_nowait(self, obj): def release(self): """ release() - No longer associate the current interpreterwith the channel + No longer associate the current interpreter with the channel (on the sending end). """ - pass + return _interpreters.channel_release(self.id) def close(self, force=False): """ close(force=False) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 2ca2d402d234d3..823af4c7527453 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -23,50 +23,86 @@ def clean_up_interpreters(): except RuntimeError: pass # already destroyed - -class LowLevelStub: - # set these as appropriate in tests - errors = () - return_create = () - ... - def __init__(self): - self._calls = [] - def _add_call(self, name, args=(), kwargs=None): - self.calls.append( - (name, args, kwargs or {})) - def _maybe_error(self): - if not self.errors: - return - err = self.errors.pop(0) - if err is not None: - raise err - def check_calls(self, test, expected): - test.assertEqual(self._calls, expected) - for returns in [self.errors, self.return_create, ...]: - test.assertEqual(tuple(returns), ()) # make sure all were used - - # the stubbed methods - def create(self): - self._add_call('create') - self._maybe_error() - return self.return_create.pop(0) - def list_all(self): - ... - def get_current(self): - ... - def get_main(self): - ... - def destroy(self, id): - ... - def run_string(self, id, text, ...): - ... - +def _run_output(interp, request, shared=None): + script, rpipe = _captured_script(request) + with rpipe: + interpreters.run_string(interp, script, shared) + return rpipe.read() class TestBase(unittest.TestCase): def tearDown(self): clean_up_interpreters() +class TestInterpreter(TestBase): + + def test_is_running(self): + interp = interpreters.Interpreter(1) + self.assertEqual(True, interp.is_running()) + + def test_destroy(self): + interp = interpreters.Interpreter(1) + interp2 = interpreters.Interpreter(2) + interp.destroy() + ids = interpreters.list_all() + self.assertEqual(ids, [interp2.id]) + + def test_run(self): + interp = interpreters.Interpreter(1) + interp.run(dedent(f""" + import _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + out = _run_output(id2, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + _interpreters.channel_release({cid}) + print(repr(obj)) + """)) + self.assertEqual(out.strip(), "b'spam'") + +class RecvChannelTest(TestBase): + + def test_release(self): + import _interpreters as interpreters + + chanl = interpreters.RecvChannel(1) + interpreters.channel_send(chanl.id, b'spam') + interpreters.channel_recv(cid) + chanl.release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close(self): + chanl = interpreters.RecvChannel(1) + chanl.close() + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(fix.cid) + +class SendChannelTest(TestBase): + + def test_release(self): + import _interpreters as interpreters + + chanl = interpreters.SendChannel(1) + interpreters.channel_send(chanl.id, b'spam') + interpreters.channel_recv(cid) + chanl.release(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close(self): + chanl = interpreters.RecvChannel(1) + chanl.close() + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(fix.cid) + class ListAllTests(TestBase): @@ -75,7 +111,6 @@ def test_initial(self): ids = interpreters.list_all() self.assertEqual(ids, [main]) - class GetCurrentTests(TestBase): def test_get_current(self): @@ -83,43 +118,44 @@ def test_get_current(self): cur = interpreters.get_current() self.assertEqual(cur, main) - -class GetMainTests(TestBase): - - def test_get_main(self): - expected, * = interpreters.list_all() - main = interpreters.get_main() - self.assertEqual(main, expected) - - class CreateTests(TestBase): def test_create(self): interp = interpreters.create() self.assertIn(interp, interpreters.list_all()) - -class DestroyTests(TestBase): - - def test_destroy(self): - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() - self.assertIn(id2, interpreters.list_all()) - interpreters.destroy(id2) - self.assertNotIn(id2, interpreters.list_all()) - - -class RunStringTests(TestBase): - - def test_run_string(self): - script, file = _captured_script('print("it worked!", end="")') - id = interpreters.create() - with file: - interpreters.run_string(id, script, None) - out = file.read() - - self.assertEqual(out, 'it worked!') +class ExceptionTests(TestBase): + + def test_does_not_exist(self): + cid = interpreters.create_channel() + recvCha = interpreters.RecvChannel(cid) + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters._channel_id(int(cid) + 1) + + def test_recv_empty(self): + cid = interpreters.create_channel() + recvCha = interpreters.RecvChannel(cid) + with self.assertRaises(interpreters.ChannelEmptyError): + recvCha.recv() + + def test_channel_not_empty(self): + cid = interpreters.create_channel() + sendCha = interpreters.SendChannel(cid) + sendCha.send(b'spam') + sendCha.send(b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + sendCha.close() + + def test_channel_closed(self): + cid = interpreters.channel_create() + sendCha = interpreters.SendChannel(cid) + sendCha.send(b'spam') + sendCha.send(b'ham') + sendCha.close() + + with self.assertRaises(interpreters.ChannelClosedError): + sendCha.send(b'spam') if __name__ == '__main__': From b7dce732b3b6f38f3fa7764700020180e867b5a5 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Tue, 3 Mar 2020 21:37:47 +0000 Subject: [PATCH 14/20] Fix tests --- Lib/interpreters.py | 107 ++++++++++----------- Lib/test/test__xxsubinterpreters.py | 98 +++++++++++++++++++ Lib/test/test_interpreters.py | 144 ++++++---------------------- Modules/_xxsubinterpretersmodule.c | 111 +++++++++++++++++++++ 4 files changed, 293 insertions(+), 167 deletions(-) diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 9fa3877540408f..97779a89390c0e 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -1,13 +1,37 @@ """Subinterpreters High Level Module.""" import _interpreters -import logger +import logging __all__ = ['Interpreter', 'SendChannel', 'RecvChannel', 'is_shareable', 'create_channel', 'list_all_channels', 'list_all', 'get_current', 'create'] +def create(): + """ create() -> Interpreter + + Initialize a new (idle) Python interpreter. + """ + id = _interpreters.create() + return Interpreter(id) + +def list_all(): + """ list_all() -> [Interpreter] + + Get all existing interpreters. + """ + return [Interpreter(id) for id in _interpreters.list_all()] + +def get_current(): + """ get_current() -> Interpreter + + Get the currently running interpreter. + """ + id = _interpreters.get_current() + return Interpreter(id) + + class Interpreter: def __init__(self, id): @@ -42,6 +66,32 @@ def run(self, src_str, /, *, channels=None): logger.error(err) raise + +def is_shareable(obj): + """ is_shareable(obj) -> Bool + + Return `True` if the object's data can be shared between + interpreters. + """ + return _interpreters.is_shareable(obj) + +def create_channel(): + """ create_channel() -> (RecvChannel, SendChannel) + + Create a new channel for passing data between interpreters. + """ + + cid = _interpreters.channel_create() + return (RecvChannel(cid), SendChannel(cid)) + +def list_all_channels(): + """ list_all_channels() -> [(RecvChannel, SendChannel)] + + Return all open channels. + """ + cid = _interpreters.channel_list_all() + return (RecvChannel(cid), SendChannel(cid)) + def wait(self, timeout): #The implementation for wait # will be non trivial to be useful @@ -55,7 +105,7 @@ class RecvChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.list_all() + self.interpreters = _interpreters.channel_list_interpreters(cid, send=False) def recv(self, timeout=2): """ channel_recv() -> obj @@ -102,7 +152,7 @@ def release(self): No longer associate the current interpreterwith the channel (on the sending end). """ - return _interpreters.(self.id) + return _interpreters(self.id) def close(self, force=False): """close(force=False) @@ -187,54 +237,3 @@ class ChannelReleasedError(ChannelClosedError): class RunFailedError(RuntimeError): pass - - -# Global API functions - -def is_shareable(obj): - """ is_shareable(obj) -> Bool - - Return `True` if the object's data can be shared between - interpreters. - """ - return _interpreters.is_shareable(obj) - -def create_channel(): - """ create_channel() -> (RecvChannel, SendChannel) - - Create a new channel for passing data between interpreters. - """ - - cid = _interpreters.channel_create() - return (RecvChannel(cid), SendChannel(cid)) - -def list_all_channels(): - """ list_all_channels() -> [(RecvChannel, SendChannel)] - - Return all open channels. - """ - cid = _interpreters.channel_list_all() - return (RecvChannel(cid), SendChannel(cid)) - -def create(): - """ create() -> Interpreter - - Initialize a new (idle) Python interpreter. - """ - id = _interpreters.create() - return Interpreter(id) - -def list_all(): - """ list_all() -> [Interpreter] - - Get all existing interpreters. - """ - return [Interpreter(id) for id in _interpreters.list_all()] - -def get_current(): - """ get_current() -> Interpreter - - Get the currently running interpreter. - """ - id = _interpreters.get_current() - return Interpreter(id) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 20e6b0419ff411..857ba48e38258a 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -1207,6 +1207,87 @@ def test_ids_global(self): self.assertEqual(cid2, int(cid1) + 1) + def test_channel_list_interpreters_none(self): + """Test listing interpreters for a channel with no associations.""" + # Test for channel with no associated interpreters. + cid = interpreters.channel_create() + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, []) + self.assertEqual(recv_interps, []) + + def test_channel_list_interpreters_basic(self): + """Test basic listing channel interpreters.""" + interp0 = interpreters.get_main() + cid = interpreters.channel_create() + interpreters.channel_send(cid, "send") + # Test for a channel that has one end associated to an interpreter. + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, [interp0]) + self.assertEqual(recv_interps, []) + + interp1 = interpreters.create() + _run_output(interp1, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + """)) + # Test for channel that has boths ends associated to an interpreter. + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, [interp0]) + self.assertEqual(recv_interps, [interp1]) + + def test_channel_list_interpreters_multiple(self): + """Test listing interpreters for a channel with many associations.""" + interp0 = interpreters.get_main() + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + cid = interpreters.channel_create() + + interpreters.channel_send(cid, "send") + _run_output(interp1, dedent(f""" + import _interpreters + obj = _interpreters.channel_send({cid}, "send") + """)) + _run_output(interp2, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + """)) + _run_output(interp3, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + """)) + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(set(send_interps), {interp0, interp1}) + self.assertEqual(set(recv_interps), {interp2, interp3}) + + @unittest.skip("Failing due to handling of destroyed interpreters") + def test_channel_list_interpreters_destroyed(self): + """Test listing channel interpreters with a destroyed interpreter.""" + interp0 = interpreters.get_main() + interp1 = interpreters.create() + cid = interpreters.channel_create() + interpreters.channel_send(cid, "send") + _run_output(interp1, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({cid}) + """)) + # Should be one interpreter associated with each end. + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, [interp0]) + self.assertEqual(recv_interps, [interp1]) + + interpreters.destroy(interp1) + # Destroyed interpreter should not be listed. + send_interps = interpreters.channel_list_interpreters(cid, send=True) + recv_interps = interpreters.channel_list_interpreters(cid, send=False) + self.assertEqual(send_interps, [interp0]) + self.assertEqual(recv_interps, []) + #################### def test_send_recv_main(self): @@ -1519,6 +1600,23 @@ def test_close_used_multiple_times_by_single_user(self): with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) + def test_channel_list_interpreters_invalid_channel(self): + cid = interpreters.channel_create() + # Test for invalid channel ID. + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_list_interpreters(1000, send=True) + + interpreters.channel_close(cid) + # Test for a channel that has been closed. + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_list_interpreters(cid, send=True) + + def test_channel_list_interpreters_invalid_args(self): + # Tests for invalid arguments passed to the API. + cid = interpreters.channel_create() + with self.assertRaises(TypeError): + interpreters.channel_list_interpreters(cid) + class ChannelReleaseTests(TestBase): diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 823af4c7527453..59c4ec0e8bf4a1 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,4 +1,5 @@ import interpreters +import _interpreters import os from textwrap import dedent import unittest @@ -15,11 +16,11 @@ def _captured_script(script): return wrapped, open(r) def clean_up_interpreters(): - for id in interpreters.list_all(): + for id in _interpreters.list_all(): if id == 0: # main continue try: - interpreters.destroy(id) + _interpreters.destroy(id) except RuntimeError: pass # already destroyed @@ -34,129 +35,46 @@ class TestBase(unittest.TestCase): def tearDown(self): clean_up_interpreters() -class TestInterpreter(TestBase): - - def test_is_running(self): - interp = interpreters.Interpreter(1) - self.assertEqual(True, interp.is_running()) +class CreateTests(TestBase): - def test_destroy(self): - interp = interpreters.Interpreter(1) - interp2 = interpreters.Interpreter(2) - interp.destroy() - ids = interpreters.list_all() - self.assertEqual(ids, [interp2.id]) + def test_create(self): + interp = interpreters.create() + lst = interpreters.list_all() + self.assertEqual(interp.id, lst[1].id) - def test_run(self): - interp = interpreters.Interpreter(1) - interp.run(dedent(f""" - import _interpreters - _interpreters.channel_send({cid}, b'spam') - """)) - out = _run_output(id2, dedent(f""" - import _interpreters - obj = _interpreters.channel_recv({cid}) - _interpreters.channel_release({cid}) - print(repr(obj)) - """)) - self.assertEqual(out.strip(), "b'spam'") - -class RecvChannelTest(TestBase): - - def test_release(self): - import _interpreters as interpreters - - chanl = interpreters.RecvChannel(1) - interpreters.channel_send(chanl.id, b'spam') - interpreters.channel_recv(cid) - chanl.release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close(self): - chanl = interpreters.RecvChannel(1) - chanl.close() - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(fix.cid) - -class SendChannelTest(TestBase): - - def test_release(self): - import _interpreters as interpreters - - chanl = interpreters.SendChannel(1) - interpreters.channel_send(chanl.id, b'spam') - interpreters.channel_recv(cid) - chanl.release(cid) - - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(cid) - - def test_close(self): - chanl = interpreters.RecvChannel(1) - chanl.close() - with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_recv(fix.cid) +class GetCurrentTests(TestBase): + def test_get_current(self): + main_interp_id = _interpreters.get_main() + cur_interp_id = interpreters.get_current().id + self.assertEqual(cur_interp_id, main_interp_id) class ListAllTests(TestBase): def test_initial(self): - main = interpreters.get_main() - ids = interpreters.list_all() - self.assertEqual(ids, [main]) + interps = interpreters.list_all() + self.assertEqual(1, len(interps)) -class GetCurrentTests(TestBase): +class TestInterpreter(TestBase): - def test_get_current(self): - main = interpreters.get_main() - cur = interpreters.get_current() - self.assertEqual(cur, main) + def test_id_fields(self): + interp = interpreters.Interpreter(1) + self.assertEqual(1, interp.id) -class CreateTests(TestBase): + def test_is_running(self): + interp_de = interpreters.create() + self.assertEqual(False, interp_de.is_running()) - def test_create(self): + def test_destroy(self): interp = interpreters.create() - self.assertIn(interp, interpreters.list_all()) - -class ExceptionTests(TestBase): - - def test_does_not_exist(self): - cid = interpreters.create_channel() - recvCha = interpreters.RecvChannel(cid) - with self.assertRaises(interpreters.ChannelNotFoundError): - interpreters._channel_id(int(cid) + 1) - - def test_recv_empty(self): - cid = interpreters.create_channel() - recvCha = interpreters.RecvChannel(cid) - with self.assertRaises(interpreters.ChannelEmptyError): - recvCha.recv() - - def test_channel_not_empty(self): - cid = interpreters.create_channel() - sendCha = interpreters.SendChannel(cid) - sendCha.send(b'spam') - sendCha.send(b'ham') - - with self.assertRaises(interpreters.ChannelNotEmptyError): - sendCha.close() - - def test_channel_closed(self): - cid = interpreters.channel_create() - sendCha = interpreters.SendChannel(cid) - sendCha.send(b'spam') - sendCha.send(b'ham') - sendCha.close() + interp2 = interpreters.create() + interp.destroy() + interps = interpreters.list_all() + self.assertEqual(2, len(interps)) - with self.assertRaises(interpreters.ChannelClosedError): - sendCha.send(b'spam') + def test_run(self): + interp = interpreters.create() + interp.run("3") + self.assertEqual(False, interp.is_running()) -if __name__ == '__main__': - unittest.main() diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 552d96a977f2a9..ff3dfb713bc50c 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -627,6 +627,27 @@ _channelends_associate(_channelends *ends, int64_t interp, int send) return 0; } +static int64_t * +_channelends_list_interpreters(_channelends *ends, int64_t *count, int send) +{ + int64_t numopen = send ? ends->numsendopen : ends->numrecvopen; + + int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)numopen); + if (ids == NULL) { + PyErr_NoMemory(); + return NULL; + } + + _channelend *ref = send ? ends->send : ends->recv; + for (int64_t i=0; ref != NULL; ref = ref->next, i++) { + ids[i] = ref->interp; + } + + *count = numopen; + + return ids; +} + static int _channelends_is_open(_channelends *ends) { @@ -1405,6 +1426,35 @@ typedef struct channelid { _channels *channels; } channelid; +static int +channel_id_converter(PyObject *arg, void *ptr) +{ + int64_t cid; + if (PyObject_TypeCheck(arg, &ChannelIDtype)) { + cid = ((channelid *)arg)->id; + } + else if (PyIndex_Check(arg)) { + cid = PyLong_AsLongLong(arg); + if (cid == -1 && PyErr_Occurred()) { + return 0; + } + if (cid < 0) { + PyErr_Format(PyExc_ValueError, + "channel ID must be a non-negative int, got %R", arg); + return 0; + } + } + else { + PyErr_Format(PyExc_TypeError, + "channel ID must be an int, got %.100s", + arg->ob_type->tp_name); + return 0; + } + *(int64_t *)ptr = cid; + return 1; +} + + static channelid * newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, int force, int resolve) @@ -2327,6 +2377,65 @@ PyDoc_STRVAR(channel_list_all_doc, \n\ Return the list of all IDs for active channels."); + +static PyObject * +channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", "send", NULL}; + int64_t cid; /* Channel ID */ + int send = 0; /* Send or receive end? */ + PyObject *ret = NULL; + + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "O&$p:channel_list_interpreters", + kwlist, channel_id_converter, &cid, &send)) { + return NULL; + } + + _PyChannelState *chan = _channels_lookup(&_globals.channels, cid, NULL); + if (chan == NULL) { + return NULL; + } + + int64_t count = 0; /* Number of interpreters */ + int64_t *ids = _channelends_list_interpreters(chan->ends, &count, send); + if (ids == NULL) { + goto except; + } + + ret = PyList_New((Py_ssize_t)count); + if (ret == NULL) { + goto except; + } + + for (int64_t i=0; i < count; i++) { + PyObject *id_obj = _PyInterpreterID_New(ids[i]); + if (id_obj == NULL) { + goto except; + } + PyList_SET_ITEM(ret, i, id_obj); + } + + goto finally; + +except: + Py_XDECREF(ret); + ret = NULL; + +finally: + PyMem_Free(ids); + return ret; +} + +PyDoc_STRVAR(channel_list_interpreters_doc, +"channel_list_interpreters(cid, *, send) -> [id]\n\ +\n\ +Return the list of all interpreter IDs associated with an end of the channel.\n\ +\n\ +The 'send' argument should be a boolean indicating whether to use the send or\n\ +receive end."); + + static PyObject * channel_send(PyObject *self, PyObject *args, PyObject *kwds) { @@ -2496,6 +2605,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channel_destroy_doc}, {"channel_list_all", channel_list_all, METH_NOARGS, channel_list_all_doc}, + {"channel_list_interpreters", (PyCFunction)(void(*)(void))channel_list_interpreters, + METH_VARARGS | METH_KEYWORDS, channel_list_interpreters_doc}, {"channel_send", (PyCFunction)(void(*)(void))channel_send, METH_VARARGS | METH_KEYWORDS, channel_send_doc}, {"channel_recv", (PyCFunction)(void(*)(void))channel_recv, From e9b56b8b954004ae9eb66b34d1019855c58ad0dd Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Wed, 4 Mar 2020 22:39:43 +0000 Subject: [PATCH 15/20] Add send buffer functionality --- Lib/interpreters.py | 104 +++++++++++++++-------------- Modules/_xxsubinterpretersmodule.c | 93 ++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 50 deletions(-) diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 97779a89390c0e..209dd44c0204b4 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -35,14 +35,18 @@ def get_current(): class Interpreter: def __init__(self, id): - self.id = id + self._id = id + + @property + def id(self): + return self._id def is_running(self): """is_running() -> bool Return whether or not the identified interpreter is running. """ - return _interpreters.is_running(self.id) + return _interpreters.is_running(self._id) def destroy(self): """destroy() @@ -52,7 +56,7 @@ def destroy(self): Attempting to destroy the current interpreter results in a RuntimeError. So does an unrecognized ID """ - return _interpreters.destroy(self.id) + return _interpreters.destroy(self._id) def run(self, src_str, /, *, channels=None): """run(src_str, /, *, channels=None) @@ -61,7 +65,7 @@ def run(self, src_str, /, *, channels=None): This blocks the current thread until done. """ try: - _interpreters.run_string(self.id, src_str) + _interpreters.run_string(self._id, src_str, channels) except RunFailedError as err: logger.error(err) raise @@ -71,7 +75,7 @@ def is_shareable(obj): """ is_shareable(obj) -> Bool Return `True` if the object's data can be shared between - interpreters. + interpreters and `False` otherwise. """ return _interpreters.is_shareable(obj) @@ -89,25 +93,21 @@ def list_all_channels(): Return all open channels. """ - cid = _interpreters.channel_list_all() - return (RecvChannel(cid), SendChannel(cid)) + return [(RecvChannel(cid), SendChannel(cid)) for cid in _interpreters.channel_list_all()] -def wait(self, timeout): +def wait(timeout): #The implementation for wait # will be non trivial to be useful import time time.sleep(timeout) -def associate_interp_to_channel(id, cid): - pass - class RecvChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.channel_list_interpreters(cid, send=False) + self.interpreters = _interpreters.channel_list_interpreters(self.id, send=False) - def recv(self, timeout=2): + def recv(self): """ channel_recv() -> obj Get the next object from the channel, @@ -117,11 +117,7 @@ def recv(self, timeout=2): obj = _interpreters.channel_recv(self.id) if obj == None: wait(timeout) - obj = obj = _interpreters.channel_recv(self.id) - - # Pending: See issue 52 on multi-core python project - associate_interp_to_channel(interpId, Cid) - + obj = _interpreters.channel_recv(self.id) return obj def recv_nowait(self, default=None): @@ -129,44 +125,32 @@ def recv_nowait(self, default=None): Like recv(), but return the default instead of waiting. """ - return _interpreters.channel_recv(self.id) - - def send_buffer(self, obj): - """ send_buffer(obj) - - Send the object's buffer to the receiving end of the channel - and wait. Associate the interpreter with the channel. - """ - pass - - def send_buffer_nowait(self, obj): - """ send_buffer_nowait(obj) - - Like send_buffer(), but return False if not received. - """ - pass + obj = _interpreters.channel_recv(self.id) + if obj == None: + obj = default + return obj def release(self): """ release() No longer associate the current interpreterwith the channel - (on the sending end). + (on the receiving end). """ - return _interpreters(self.id) + return _interpreters.channel_release(self.id, recv=True) def close(self, force=False): """close(force=False) Close the channel in all interpreters.. """ - return _interpreters.channel_close(self.id, force) + return _interpreters.channel_close(self.id, recv=force) class SendChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.list_all() + self.interpreters = _interpreters.channel_list_interpreters(self.id, send=True) def send(self, obj): """ send(obj) @@ -174,37 +158,57 @@ def send(self, obj): Send the object (i.e. its data) to the receiving end of the channel and wait. Associate the interpreter with the channel. """ - obj = _interpreters.channel_send(self.id, obj) + _interpreters.channel_send(self.id, obj) wait(2) - associate_interp_to_channel(interpId, Cid) def send_nowait(self, obj): """ send_nowait(obj) Like send(), but return False if not received. """ - try: - obj = _interpreters.channel_send(self.id, obj) - except: + _interpreters.channel_send(self.id, obj) + recv_obj = _interpreters.channel_recv(self.id) + if recv_obj: + return obj + else: return False - return obj + def send_buffer(self, obj): + """ ssend_buffer(obj) + + Send the object's buffer to the receiving + end of the channel and wait. Associate the interpreter + with the channel. + """ + _interpreters.channel_send_buffer(self.id, obj) + wait(2) + + def send_buffer_nowait(self, obj): + """ send_buffer_nowait(obj) + + Like send(), but return False if not received. + """ + _interpreters.channel_send_buffer(self.id, obj) + recv_obj = _interpreters.channel_recv(self.id) + if recv_obj: + return obj + else: + return False def release(self): """ release() - No longer associate the current interpreter with the channel + No longer associate the current interpreterwith the channel (on the sending end). """ - return _interpreters.channel_release(self.id) + return _interpreters.channel_release(self.id, send=True) def close(self, force=False): - """ close(force=False) + """close(force=False) - No longer associate the current interpreterwith the channel - (on the sending end). + Close the channel in all interpreters.. """ - return _interpreters.channel_close(self.id, force) + return _interpreters.channel_close(self.id, send=force) class ChannelError(Exception): diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index ff3dfb713bc50c..894e0641aa48ca 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1303,6 +1303,71 @@ _channel_destroy(_channels *channels, int64_t id) return 0; } +static int +_channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) +{ + const char *s = NULL; + Py_buffer view = {NULL, NULL}; + if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0){ + return -1; + } + + s = view.buf; + if (s == NULL) { + PyBuffer_Release(&view); + return -1; + } + + PyInterpreterState *interp = _get_current(); + if (interp == NULL) { + PyBuffer_Release(&view); + return -1; + } + + // Look up the channel. + PyThread_type_lock mutex = NULL; + _PyChannelState *chan = _channels_lookup(channels, id, &mutex); + if (chan == NULL) { + PyBuffer_Release(&view); + return -1; + } + // Past this point we are responsible for releasing the mutex. + + if (chan->closing != NULL) { + PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", id); + PyThread_release_lock(mutex); + PyBuffer_Release(&view); + return -1; + } + + // Convert the buffer to cross-interpreter data. + _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); + if (data == NULL) { + PyThread_release_lock(mutex); + PyBuffer_Release(&view); + return -1; + } + if (_PyObject_GetCrossInterpreterData((PyObject *)s, data) != 0) { + PyThread_release_lock(mutex); + PyMem_Free(data); + PyBuffer_Release(&view); + return -1; + } + + // Add the data to the channel. + int res = _channel_add(chan, PyInterpreterState_GetID(interp), data); + PyThread_release_lock(mutex); + if (res != 0) { + _PyCrossInterpreterData_Release(data); + PyMem_Free(data); + PyBuffer_Release(&view); + return -1; + } + + PyBuffer_Release(&view); + return 0; +} + static int _channel_send(_channels *channels, int64_t id, PyObject *obj) { @@ -2462,6 +2527,32 @@ PyDoc_STRVAR(channel_send_doc, \n\ Add the object's data to the channel's queue."); +static PyObject * +channel_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", "obj", NULL}; + PyObject *id; + PyObject *obj; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO:channel_send_buffer", kwlist, &id, &obj)) { + return NULL; + } + int64_t cid = _Py_CoerceID(id); + if (cid < 0) { + return NULL; + } + + if (_channel_send_buffer(&_globals.channels, cid, obj) != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_send_buffer_doc, +"channel_send_buffer(cid, obj)\n\ +\n\ +Add the object's buffer to the channel's queue."); + static PyObject * channel_recv(PyObject *self, PyObject *args, PyObject *kwds) { @@ -2609,6 +2700,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channel_list_interpreters_doc}, {"channel_send", (PyCFunction)(void(*)(void))channel_send, METH_VARARGS | METH_KEYWORDS, channel_send_doc}, + {"channel_send_buffer", (PyCFunction)(void(*)(void))channel_send_buffer, + METH_VARARGS | METH_KEYWORDS, channel_send_buffer_doc}, {"channel_recv", (PyCFunction)(void(*)(void))channel_recv, METH_VARARGS | METH_KEYWORDS, channel_recv_doc}, {"channel_close", (PyCFunction)(void(*)(void))channel_close, From 2bf89d41507aab7e3daec1b558e6b69c19067913 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Thu, 5 Mar 2020 19:46:39 +0000 Subject: [PATCH 16/20] Add tests for Global functions and Interpreter --- Doc/library/interpreters.rst | 9 +- Lib/interpreters.py | 14 +- Lib/test/test_interpreters.py | 454 ++++++++++++++++++++++++++++++++-- 3 files changed, 453 insertions(+), 24 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 5d15484b6b4d78..3944b8d0712a42 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -8,9 +8,9 @@ -------------- -This module provides tools for working with sub-interpreters, such as creating them, -running code in them, or sending data between them. It is a wrapper around the low- -level :mod:`_interpreters` module. +This module provides highlevel tools for working with sub-interpreters, +such as creating them, running code in them, or sending data between them. +It is a wrapper around the low-level :mod:`_interpreters` module. .. versionchanged:: added in 3.9 @@ -124,7 +124,8 @@ This module defines the following global functions: .. function:: get_current() - Get the currently running interpreter. + Get the currently running interpreter. This method returns + an `interpreter` object. .. function:: list_all() diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 209dd44c0204b4..10591c2bea0160 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -61,11 +61,23 @@ def destroy(self): def run(self, src_str, /, *, channels=None): """run(src_str, /, *, channels=None) + channel = (RecvChannel, SendChannel) + Run the given source code in the interpreter. This blocks the current thread until done. """ + if channels: + if channels[0] and channels[1] != None: + _interpreters.channel_recv(channels[0].id) + _interpreters.channel_send(channels[1].id, src_str) + elif channels[0] != None and channels[1] == None: + _interpreters.channel_recv(channels[0].id) + elif channels[0] == None and channels[1] != None: + _interpreters.channel_send(channels[1].id, src_str) + else: + pass try: - _interpreters.run_string(self._id, src_str, channels) + _interpreters.run_string(self._id, src_str) except RunFailedError as err: logger.error(err) raise diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 59c4ec0e8bf4a1..0253dce6a04e89 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,9 +1,12 @@ -import interpreters -import _interpreters +import contextlib import os +import threading from textwrap import dedent import unittest +import interpreters +import _interpreters + def _captured_script(script): r, w = os.pipe() indented = script.replace('\n', '\n ') @@ -16,65 +19,478 @@ def _captured_script(script): return wrapped, open(r) def clean_up_interpreters(): - for id in _interpreters.list_all(): - if id == 0: # main + for interp in interpreters.list_all(): + if interp.id == 0: # main continue try: - _interpreters.destroy(id) + interp.destroy() except RuntimeError: pass # already destroyed def _run_output(interp, request, shared=None): script, rpipe = _captured_script(request) with rpipe: - interpreters.run_string(interp, script, shared) + interp.run(script) return rpipe.read() +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interp.run(dedent(f""" + # wait for "signal" + with open({r}) as rpipe: + rpipe.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as spipe: + spipe.write('done') + t.join() + + class TestBase(unittest.TestCase): def tearDown(self): clean_up_interpreters() + class CreateTests(TestBase): - def test_create(self): + def test_in_main(self): interp = interpreters.create() lst = interpreters.list_all() self.assertEqual(interp.id, lst[1].id) + def test_in_thread(self): + lock = threading.Lock() + id = None + interp = interpreters.create() + lst = interpreters.list_all() + def f(): + nonlocal id + id = interp.id + lock.acquire() + lock.release() + + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertEqual(interp.id, lst[1].id) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + interp = interpreters.create() + print(interp) + """)) + interp2 = out.strip() + + self.assertEqual(len(set(interpreters.list_all())), len({main, interp, interp2})) + + def test_in_threaded_subinterpreter(self): + main, = interpreters.list_all() + interp = interpreters.create() + interp2 = None + def f(): + nonlocal interp2 + out = _run_output(interp, dedent(""" + import interpreters + interp = interpreters.create() + print(interp) + """)) + interp2 = int(out.strip()) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assertEqual(len(set(interpreters.list_all())), len({main, interp, interp2})) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp_lst = [] + for _ in range(3): + interps = interpreters.create() + interp_lst.append(interps) + # Now destroy them. + for interp in interp_lst: + interp.destroy() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(len(set(interpreters.list_all())), len(before | {interp})) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + interp1 = interpreters.create() + interp2 = interpreters.create() + interp3 = interpreters.create() + # Now destroy 2 of them. + interp1.destroy() + interp2.destroy() + # Finally, create another. + interp = interpreters.create() + self.assertEqual(len(set(interpreters.list_all())), len(before | {interp3, interp})) + + class GetCurrentTests(TestBase): - def test_get_current(self): + def test_main(self): main_interp_id = _interpreters.get_main() cur_interp_id = interpreters.get_current().id self.assertEqual(cur_interp_id, main_interp_id) + def test_subinterpreter(self): + main = _interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + cur = interpreters.get_current() + print(cur) + """)) + cur = out.strip() + self.assertNotEqual(cur, main) + + class ListAllTests(TestBase): def test_initial(self): interps = interpreters.list_all() self.assertEqual(1, len(interps)) -class TestInterpreter(TestBase): + def test_after_creating(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, first.id, second.id]) + + def test_after_destroying(self): + main = interpreters.get_current() + first = interpreters.create() + second = interpreters.create() + first.destroy() + + ids = [] + for interp in interpreters.list_all(): + ids.append(interp.id) + + self.assertEqual(ids, [main.id, second.id]) + + +class TestInterpreterId(TestBase): - def test_id_fields(self): + def test_id_field_in_main(self): + main = interpreters.get_current() + self.assertEqual(0, main.id) + + def test_id_field_custom(self): interp = interpreters.Interpreter(1) self.assertEqual(1, interp.id) - def test_is_running(self): - interp_de = interpreters.create() - self.assertEqual(False, interp_de.is_running()) + def test_id_field_readonly(self): + interp = interpreters.Interpreter(1) + with self.assertRaises(AttributeError): + interp.id = 2 + + +class TestInterpreterIsRunning(TestBase): + + def test_is_running_main(self): + main = interpreters.get_current() + self.assertTrue(main.is_running()) + + def test_is_running_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interp.is_running()) + + with _running(interp): + self.assertTrue(interp.is_running()) + self.assertFalse(interp.is_running()) + + def test_is_running_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _interpreters + if _interpreters.is_running({interp.id}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') - def test_destroy(self): + def test_is_running_already_destroyed(self): interp = interpreters.create() + interp.destroy() + with self.assertRaises(RuntimeError): + interp.is_running() + + def test_is_running_bad_id(self): + interp = interpreters.Interpreter(-1) + with self.assertRaises(RuntimeError): + interp.is_running() + + +class TestInterpreterDestroy(TestBase): + + def test_destroy_basic(self): + interp1 = interpreters.create() interp2 = interpreters.create() + interp3 = interpreters.create() + self.assertEqual(4, len(interpreters.list_all())) + interp2.destroy() + self.assertEqual(3, len(interpreters.list_all())) + + def test_destroy_all(self): + before = set(interpreters.list_all()) + interps = set() + for _ in range(3): + interp = interpreters.create() + interps.add(interp) + self.assertEqual(len(set(interpreters.list_all())), len(before | interps)) + for interp in interps: + interp.destroy() + self.assertEqual(len(set(interpreters.list_all())), len(before)) + + def test_destroy_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + main.destroy() + + def f(): + with self.assertRaises(RuntimeError): + main.destroy() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_destroy_already_destroyed(self): + interp = interpreters.create() interp.destroy() - interps = interpreters.list_all() - self.assertEqual(2, len(interps)) + with self.assertRaises(RuntimeError): + interp.destroy() - def test_run(self): + def test_destroy_from_current(self): + main, = interpreters.list_all() interp = interpreters.create() - interp.run("3") - self.assertEqual(False, interp.is_running()) + script = dedent(f""" + import interpreters + try: + main = interpreters.get_current() + main.destroy() + except RuntimeError: + pass + """) + + interp.run(script) + self.assertEqual(len(set(interpreters.list_all())), len({main, interp})) + + def test_destroy_from_sibling(self): + main, = interpreters.list_all() + interp1 = interpreters.create() + script = dedent(f""" + import interpreters + interp2 = interpreters.create() + interp2.destroy() + """) + interp1.run(script) + + self.assertEqual(len(set(interpreters.list_all())), len({main, interp1})) + + def test_destroy_from_other_thread(self): + interp = interpreters.create() + def f(): + interp.destroy() + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_destroy_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interp.destroy() + self.assertTrue(interp.is_running()) + + +class TestInterpreterRun(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + super().setUp() + self.interp = interpreters.create() + self._fs = None + + def tearDown(self): + if self._fs is not None: + self._fs.close() + super().tearDown() + + @property + def fs(self): + if self._fs is None: + self._fs = FSFixture(self) + return self._fs + + def test_success(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + self.interp.run(script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_in_thread(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + self.interp.run(script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_create_thread(self): + script, file = _captured_script(""" + import threading + def f(): + print('it worked!', end='') + + t = threading.Thread(target=f) + t.start() + t.join() + """) + with file: + self.interp.run(script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + import tempfile + with tempfile.NamedTemporaryFile('w+') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + import os + try: + os.fork() + except RuntimeError: + with open('{file.name}', 'w') as out: + out.write('{expected}') + """) + self.interp.run(script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + def test_already_running(self): + with _running(self.interp): + with self.assertRaises(RuntimeError): + self.interp.run('print("spam")') + + def test_bad_script(self): + with self.assertRaises(TypeError): + self.interp.run(10) + + def test_bytes_for_script(self): + with self.assertRaises(TypeError): + self.interp.run(b'print("spam")') + + +class TestIsShareable(TestBase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + 'spam', + 10, + -10, + ] + for obj in shareables: + with self.subTest(obj): + self.assertTrue( + interpreters.is_shareable(obj)) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + True, + False, + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + 100.0, + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(repr(obj)): + self.assertFalse( + interpreters.is_shareable(obj)) + + +class TestChannel(TestBase): + + def test_create_cid(self): + r, s = interpreters.create_channel() + self.assertIsInstance(r, interpreters.RecvChannel) + self.assertIsInstance(s, interpreters.SendChannel) + + def test_sequential_ids(self): + before = interpreters.list_all_channels() + channels1 = interpreters.create_channel() + channels2 = interpreters.create_channel() + channels3 = interpreters.create_channel() + after = interpreters.list_all_channels() + self.assertEqual(len(set(after) - set(before)), + len({channels1, channels2, channels3})) +class TestRecvChannelID(TestBase): \ No newline at end of file From 158e54fe848308287c66c6cc65683e46b3aae990 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye Date: Fri, 6 Mar 2020 21:35:43 +0000 Subject: [PATCH 17/20] Test coverage and documentation --- Doc/library/interpreters.rst | 92 ++++++---- Lib/interpreters.py | 177 +++++++++++++----- Lib/test/test_interpreters.py | 283 ++++++++++++++++++++++++++--- Modules/_xxsubinterpretersmodule.c | 84 +++++++-- 4 files changed, 515 insertions(+), 121 deletions(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 3944b8d0712a42..6b9c35b6250b89 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -8,7 +8,7 @@ -------------- -This module provides highlevel tools for working with sub-interpreters, +This module provides highlevel tools for working with sub-interpreters, such as creating them, running code in them, or sending data between them. It is a wrapper around the low-level :mod:`_interpreters` module. @@ -25,16 +25,18 @@ The Interpreter object represents a single interpreter. .. method:: is_running() Return whether or not the identified interpreter is running. + It returns `True` and `False` otherwise. .. method:: destroy() - Destroy the identified interpreter. Attempting to destroy the current - interpreter results in a RuntimeError. So does an unrecognized ID. + Destroy the interpreter. Attempting to destroy the current + interpreter results in a `RuntimeError`. .. method:: run(self, src_str, /, *, channels=None): - Run the given source code in the interpreter. This blocks the current - thread until done. + Run the given source code in the interpreter. This blocks + the current thread until done. `channels` should be in + the form : `(RecvChannel, SendChannel)`. RecvChannel Objects ------------------- @@ -47,60 +49,69 @@ The RecvChannel object represents a recieving channel. .. method:: recv() - Get the next object from the channel, and wait if none have been - sent. Associate the interpreter with the channel. + Get the next object from the channel, and wait if + none have been sent. Associate the interpreter + with the channel. .. method:: recv_nowait(default=None) - Like ``recv()``, but return the default instead of waiting. + Like ``recv()``, but return the default result + instead of waiting. - .. method:: release() + .. method:: release() - Close the channel for the current interpreter. 'send' and 'recv' (bool) may - be used to indicate the ends to close. By default both ends are closed. - Closing an already closed end is a noop. + Release the channel for the current interpreter. + By default both ends are released. Releasing an already + released end results in a ``ChannelReleasedError`` exception. .. method:: close(force=False) - Close the channel in all interpreters. + Close the channel in all interpreters. By default + both ends are closed. closing an already closed end + results in a ``ChannelClosedError`` exception. Without + seeting ``force`` to ``True`` a ``ChannelNotEmptyError`` + will be returned when a channel with data is closed. SendChannel Objects -------------------- -The SendChannel object represents a sending channel. +The ``SendChannel`` object represents a sending channel. .. class:: SendChannel(id) - This class represents the receiving end of a channel. + This class represents the sending end of a channel. .. method:: send(obj) - Send the object (i.e. its data) to the receiving end of the channel - and wait.Associate the interpreter with the channel. + Send the object ``obj`` to the receiving end of the channel + and wait. Associate the interpreter with the channel. .. method:: send_nowait(obj) - Like ``send()``, but return False if not received. + Like ``send()`` but return ``False`` if not received. .. method:: send_buffer(obj) - Send the object's buffer to the receiving end of the channel and wait. - Associate the interpreter with the channel. + Send the object's buffer to the receiving end of the + channel and wait. Associate the interpreter with the + channel. .. method:: send_buffer_nowait(obj) - Like ``send_buffer()``, but return False if not received. + Like ``send_buffer()`` but return ``False`` if not received. .. method:: release() - Close the channel for the current interpreter. 'send' and 'recv' (bool) may - be used to indicate the ends to close. By default both ends are closed. - Closing an already closed end is a noop. + Release the channel for the current interpreter. + By default both ends are released. Releasing an already + released end results in a ``ChannelReleasedError`` exception. .. method:: close(force=False) - Close the channel in all interpreters. + Close the channel in all interpreters. By default + both ends are closed. closing an already closed end + results in a ``ChannelClosedError`` exception. This module defines the following global functions: @@ -108,7 +119,8 @@ This module defines the following global functions: .. function:: is_shareable(obj) - Return `True` if the object's data can be shared between interpreters. + Return ``True`` if the object's data can be shared between + interpreters. .. function:: create_channel() @@ -120,16 +132,18 @@ This module defines the following global functions: .. function:: create() - Initialize a new (idle) Python interpreter. + Initialize a new (idle) Python interpreter. Get the currently + running interpreter. This method returns an ``Interpreter`` object. .. function:: get_current() Get the currently running interpreter. This method returns - an `interpreter` object. + an ``Interpreter`` object. .. function:: list_all() - Get all existing interpreters. + Get all existing interpreters. Returns a list + of ``Interpreter`` objects. This module also defines the following exceptions. @@ -140,35 +154,35 @@ This module also defines the following exceptions. .. exception:: ChannelError - This exception, a subclass of :exc:`Exception`, and is the base class for - channel-related exceptions. + This exception is a subclass of :exc:`Exception`, and is the base + class for all channel-related exceptions. .. exception:: ChannelNotFoundError - This exception, a subclass of :exc:`ChannelError`, is raised when the - the identified channel was not found. + This exception is a subclass of :exc:`ChannelError`, and is raised + when the the identified channel is not found. .. exception:: ChannelEmptyError - This exception, a subclass of :exc:`ChannelError`, is raised when + This exception is a subclass of :exc:`ChannelError`, and is raised when the channel is unexpectedly empty. .. exception:: ChannelNotEmptyError - This exception, a subclass of :exc:`ChannelError`, is raised when + This exception is a subclass of :exc:`ChannelError`, and is raised when the channel is unexpectedly not empty. .. exception:: NotReceivedError - This exception, a subclass of :exc:`ChannelError`, is raised when + This exception is a subclass of :exc:`ChannelError`, and is raised when nothing was waiting to receive a sent object. .. exception:: ChannelClosedError - This exception, a subclass of :exc:`ChannelError`, is raised when + This exception is a subclass of :exc:`ChannelError`, and is raised when the channel is closed. .. exception:: ChannelReleasedError - This exception, a subclass of :exc:`ChannelClosedError`, is raised when - the channel is released (but not yet closed). + This exception is a subclass of :exc:`ChannelClosedError`, and is raised + when the channel is released (but not yet closed). diff --git a/Lib/interpreters.py b/Lib/interpreters.py index 10591c2bea0160..04e9d551a547dc 100644 --- a/Lib/interpreters.py +++ b/Lib/interpreters.py @@ -3,8 +3,9 @@ import _interpreters import logging -__all__ = ['Interpreter', 'SendChannel', 'RecvChannel', 'is_shareable', - 'create_channel', 'list_all_channels', 'list_all', 'get_current', +__all__ = ['Interpreter', 'SendChannel', 'RecvChannel', + 'is_shareable', 'create_channel', + 'list_all_channels', 'get_current', 'create'] @@ -21,7 +22,8 @@ def list_all(): Get all existing interpreters. """ - return [Interpreter(id) for id in _interpreters.list_all()] + return [Interpreter(id) for id in + _interpreters.list_all()] def get_current(): """ get_current() -> Interpreter @@ -44,28 +46,24 @@ def id(self): def is_running(self): """is_running() -> bool - Return whether or not the identified interpreter is running. + Return whether or not the identified + interpreter is running. """ return _interpreters.is_running(self._id) def destroy(self): """destroy() - Destroy the identified interpreter. + Destroy the interpreter. Attempting to destroy the current - interpreter results in a RuntimeError. So does an unrecognized ID + interpreter results in a RuntimeError. """ return _interpreters.destroy(self._id) - def run(self, src_str, /, *, channels=None): - """run(src_str, /, *, channels=None) - - channel = (RecvChannel, SendChannel) - - Run the given source code in the interpreter. - This blocks the current thread until done. - """ + def _handle_channels(self, channels): + # Looks like the only way for an interpreter to be associated + # to a channel is through sending or recving data. if channels: if channels[0] and channels[1] != None: _interpreters.channel_recv(channels[0].id) @@ -76,6 +74,14 @@ def run(self, src_str, /, *, channels=None): _interpreters.channel_send(channels[1].id, src_str) else: pass + + def run(self, src_str, /, *, channels=None): + """run(src_str, /, *, channels=None) + + Run the given source code in the interpreter. + This blocks the current thread until done. + """ + self._handle_channels(channels) try: _interpreters.run_string(self._id, src_str) except RunFailedError as err: @@ -86,15 +92,16 @@ def run(self, src_str, /, *, channels=None): def is_shareable(obj): """ is_shareable(obj) -> Bool - Return `True` if the object's data can be shared between - interpreters and `False` otherwise. + Return `True` if the object's data can be + shared between interpreters and `False` otherwise. """ return _interpreters.is_shareable(obj) def create_channel(): """ create_channel() -> (RecvChannel, SendChannel) - Create a new channel for passing data between interpreters. + Create a new channel for passing data between + interpreters. """ cid = _interpreters.channel_create() @@ -105,7 +112,8 @@ def list_all_channels(): Return all open channels. """ - return [(RecvChannel(cid), SendChannel(cid)) for cid in _interpreters.channel_list_all()] + return [(RecvChannel(cid), SendChannel(cid)) + for cid in _interpreters.channel_list_all()] def wait(timeout): #The implementation for wait @@ -117,7 +125,9 @@ class RecvChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.channel_list_interpreters(self.id, send=False) + self.interpreters = _interpreters.\ + channel_list_interpreters(self.id,\ + send=False) def recv(self): """ channel_recv() -> obj @@ -126,101 +136,174 @@ def recv(self): and wait if none have been sent. Associate the interpreter with the channel. """ - obj = _interpreters.channel_recv(self.id) - if obj == None: - wait(timeout) + try: obj = _interpreters.channel_recv(self.id) + if obj == None: + wait(2) + obj = _interpreters.channel_recv(self.id) + except _interpreters.ChannelEmptyError: + raise ChannelEmptyError + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError return obj def recv_nowait(self, default=None): """recv_nowait(default=None) -> object - Like recv(), but return the default instead of waiting. + Like recv(), but return the default + instead of waiting. """ - obj = _interpreters.channel_recv(self.id) - if obj == None: - obj = default + try: + obj = _interpreters.channel_recv(self.id) + if obj == None: + obj = default + except _interpreters.ChannelEmptyError: + raise ChannelEmptyError + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError return obj def release(self): """ release() - No longer associate the current interpreterwith the channel - (on the receiving end). + No longer associate the current interpreter + with the channel (on the receiving end). """ - return _interpreters.channel_release(self.id, recv=True) + try: + _interpreters.channel_release(self.id, recv=True) + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.ChannelNotEmptyError: + raise ChannelNotEmptyError def close(self, force=False): """close(force=False) - Close the channel in all interpreters.. + Close the channel in all interpreters. """ - return _interpreters.channel_close(self.id, recv=force) + try: + _interpreters.channel_close(self.id, + force=force, recv=True) + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.ChannelNotEmptyError: + raise ChannelNotEmptyError class SendChannel: def __init__(self, id): self.id = id - self.interpreters = _interpreters.channel_list_interpreters(self.id, send=True) + self.interpreters = _interpreters.\ + channel_list_interpreters(self.id,\ + send=True) def send(self, obj): """ send(obj) - Send the object (i.e. its data) to the receiving end of the channel - and wait. Associate the interpreter with the channel. + Send the object (i.e. its data) to the receiving + end of the channel and wait. Associate the interpreter + with the channel. """ - _interpreters.channel_send(self.id, obj) - wait(2) + try: + _interpreters.channel_send(self.id, obj) + wait(2) + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError def send_nowait(self, obj): """ send_nowait(obj) Like send(), but return False if not received. """ - _interpreters.channel_send(self.id, obj) + try: + _interpreters.channel_send(self.id, obj) + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError + recv_obj = _interpreters.channel_recv(self.id) if recv_obj: - return obj + return obj else: return False def send_buffer(self, obj): - """ ssend_buffer(obj) + """ send_buffer(obj) Send the object's buffer to the receiving end of the channel and wait. Associate the interpreter with the channel. """ - _interpreters.channel_send_buffer(self.id, obj) - wait(2) + try: + _interpreters.channel_send_buffer(self.id, obj) + wait(2) + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError def send_buffer_nowait(self, obj): """ send_buffer_nowait(obj) Like send(), but return False if not received. """ - _interpreters.channel_send_buffer(self.id, obj) + try: + _interpreters.channel_send_buffer(self.id, obj) + except _interpreters.ChannelNotFoundError: + raise ChannelNotFoundError + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.RunFailedError: + raise RunFailedError recv_obj = _interpreters.channel_recv(self.id) if recv_obj: - return obj + return obj else: return False def release(self): """ release() - No longer associate the current interpreterwith the channel - (on the sending end). + No longer associate the current interpreter + with the channel (on the sending end). """ - return _interpreters.channel_release(self.id, send=True) + try: + _interpreters.channel_release(self.id, send=True) + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.ChannelNotEmptyError: + raise ChannelNotEmptyError def close(self, force=False): """close(force=False) Close the channel in all interpreters.. """ - return _interpreters.channel_close(self.id, send=force) + try: + _interpreters.channel_close(self.id, + force=force, send=False) + except _interpreters.ChannelClosedError: + raise ChannelClosedError + except _interpreters.ChannelNotEmptyError: + raise ChannelNotEmptyError class ChannelError(Exception): diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 0253dce6a04e89..7570b925254ba8 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -3,7 +3,7 @@ import threading from textwrap import dedent import unittest - +import time import interpreters import _interpreters @@ -107,11 +107,11 @@ def f(): print(interp) """)) interp2 = int(out.strip()) - + t = threading.Thread(target=f) t.start() t.join() - + self.assertEqual(len(set(interpreters.list_all())), len({main, interp, interp2})) def test_after_destroy_all(self): @@ -193,15 +193,15 @@ def test_after_destroying(self): class TestInterpreterId(TestBase): - def test_id_field_in_main(self): + def test_in_main(self): main = interpreters.get_current() self.assertEqual(0, main.id) - def test_id_field_custom(self): + def test_with_custom_num(self): interp = interpreters.Interpreter(1) self.assertEqual(1, interp.id) - def test_id_field_readonly(self): + def test_for_readonly_property(self): interp = interpreters.Interpreter(1) with self.assertRaises(AttributeError): interp.id = 2 @@ -209,11 +209,11 @@ def test_id_field_readonly(self): class TestInterpreterIsRunning(TestBase): - def test_is_running_main(self): + def test_main(self): main = interpreters.get_current() self.assertTrue(main.is_running()) - def test_is_running_subinterpreter(self): + def test_subinterpreter(self): interp = interpreters.create() self.assertFalse(interp.is_running()) @@ -221,7 +221,7 @@ def test_is_running_subinterpreter(self): self.assertTrue(interp.is_running()) self.assertFalse(interp.is_running()) - def test_is_running_from_subinterpreter(self): + def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" import _interpreters @@ -232,13 +232,13 @@ def test_is_running_from_subinterpreter(self): """)) self.assertEqual(out.strip(), 'True') - def test_is_running_already_destroyed(self): + def test_already_destroyed(self): interp = interpreters.create() interp.destroy() with self.assertRaises(RuntimeError): interp.is_running() - def test_is_running_bad_id(self): + def test_bad_id(self): interp = interpreters.Interpreter(-1) with self.assertRaises(RuntimeError): interp.is_running() @@ -246,7 +246,7 @@ def test_is_running_bad_id(self): class TestInterpreterDestroy(TestBase): - def test_destroy_basic(self): + def test_basic(self): interp1 = interpreters.create() interp2 = interpreters.create() interp3 = interpreters.create() @@ -254,7 +254,7 @@ def test_destroy_basic(self): interp2.destroy() self.assertEqual(3, len(interpreters.list_all())) - def test_destroy_all(self): + def test_all(self): before = set(interpreters.list_all()) interps = set() for _ in range(3): @@ -265,7 +265,7 @@ def test_destroy_all(self): interp.destroy() self.assertEqual(len(set(interpreters.list_all())), len(before)) - def test_destroy_main(self): + def test_main(self): main, = interpreters.list_all() with self.assertRaises(RuntimeError): main.destroy() @@ -278,13 +278,13 @@ def f(): t.start() t.join() - def test_destroy_already_destroyed(self): + def test_already_destroyed(self): interp = interpreters.create() interp.destroy() with self.assertRaises(RuntimeError): interp.destroy() - def test_destroy_from_current(self): + def test_from_current(self): main, = interpreters.list_all() interp = interpreters.create() script = dedent(f""" @@ -299,7 +299,7 @@ def test_destroy_from_current(self): interp.run(script) self.assertEqual(len(set(interpreters.list_all())), len({main, interp})) - def test_destroy_from_sibling(self): + def test_from_sibling(self): main, = interpreters.list_all() interp1 = interpreters.create() script = dedent(f""" @@ -311,7 +311,7 @@ def test_destroy_from_sibling(self): self.assertEqual(len(set(interpreters.list_all())), len({main, interp1})) - def test_destroy_from_other_thread(self): + def test_from_other_thread(self): interp = interpreters.create() def f(): interp.destroy() @@ -320,7 +320,7 @@ def f(): t.start() t.join() - def test_destroy_still_running(self): + def test_still_running(self): main, = interpreters.list_all() interp = interpreters.create() with _running(interp): @@ -490,7 +490,248 @@ def test_sequential_ids(self): channels3 = interpreters.create_channel() after = interpreters.list_all_channels() - self.assertEqual(len(set(after) - set(before)), + self.assertEqual(len(set(after) - set(before)), len({channels1, channels2, channels3})) -class TestRecvChannelID(TestBase): \ No newline at end of file +class TestSendRecv(TestBase): + + def test_fields(self): + r, s = interpreters.create_channel() + self.assertGreaterEqual(r.id, 0) + self.assertEqual([], r.interpreters) + + def test_send_recv_main(self): + r, s = interpreters.create_channel() + orig = b'spam' + s.send(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + r, s = interpreters.create_channel() + orig = b'spam' + s.send(orig) + obj = r.recv() + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_threads(self): + r, s = interpreters.create_channel() + + def f(): + while True: + try: + obj = r.recv() + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + s.send(b'spam') + t.join() + obj = r.recv() + + self.assertEqual(obj, b'spam') + + def test_recv_empty(self): + r, s = interpreters.create_channel() + with self.assertRaises(interpreters.ChannelEmptyError): + r.recv() + + def test_send_recv_nowait_main(self): + r, s = interpreters.create_channel() + orig = b'spam' + s.send(orig) + obj = r.recv_nowait() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_nowait_same_interpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + r, s = interpreters.create_channel() + orig = b'spam' + s.send(orig) + obj = r.recv_nowait() + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_nowait_different_threads(self): + r, s = interpreters.create_channel() + + def f(): + while True: + try: + obj = r.recv_nowait() + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + s.send(obj) + t = threading.Thread(target=f) + t.start() + + s.send(b'spam') + t.join() + obj = r.recv_nowait() + + self.assertEqual(obj, b'spam') + + def test_recv_nowait_empty(self): + r, s = interpreters.create_channel() + with self.assertRaises(interpreters.ChannelEmptyError): + r.recv_nowait() + + # close + + def test_close_single_user(self): + r, s = interpreters.create_channel() + s.send(b'spam') + r.recv() + s.close() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + def test_close_multiple_times(self): + r, s = interpreters.create_channel() + s.send(b'spam') + r.recv() + s.close() + + with self.assertRaises(interpreters.ChannelClosedError): + s.close() + + def test_close_empty(self): + tests = [ + (False, False), + (True, False), + (False, True), + (True, True), + ] + for send, recv in tests: + with self.subTest((send, recv)): + r, s = interpreters.create_channel() + s.send(b'spam') + r.recv() + s.close() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + def test_close_defaults_with_unused_items(self): + r, s = interpreters.create_channel() + s.send(b'spam') + s.send(b'ham') + + with self.assertRaises(interpreters.ChannelNotEmptyError): + s.close() + r.recv() + s.send(b'eggs') + + def test_close_never_used(self): + r, s = interpreters.create_channel() + r.close() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + def test_close_used_multiple_times_by_single_user(self): + r, s = interpreters.create_channel() + s.send(b'spam') + s.send(b'spam') + s.send(b'spam') + r.recv() + s.close(force=True) + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + # release + + def test_single_user(self): + r, s = interpreters.create_channel() + s.send(b'spam') + r.recv() + s.release() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'eggs') + + def test_with_unused_items(self): + r, s = interpreters.create_channel() + s.send(b'spam') + s.send(b'ham') + s.release() + + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + + def test_never_used(self): + r, s = interpreters.create_channel() + s.release() + + with self.assertRaises(interpreters.ChannelClosedError): + s.send(b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + r.recv() + +class TestSendBuffer(TestBase): + def test_send_recv_main(self): + r, s = interpreters.create_channel() + orig = b'spam' + s.send_buffer(orig) + obj = r.recv() + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import interpreters + r, s = interpreters.create_channel() + orig = b'spam' + s.send_buffer(orig) + obj = r.recv() + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_threads(self): + r, s = interpreters.create_channel() + + def f(): + while True: + try: + obj = r.recv() + break + except interpreters.ChannelEmptyError: + time.sleep(0.1) + s.send_buffer(obj) + t = threading.Thread(target=f) + t.start() + + s.send_buffer(b'spam') + t.join() + obj = r.recv() + + self.assertEqual(obj, b'spam') diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 894e0641aa48ca..008517c0a1d407 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -286,6 +286,8 @@ static PyObject *ChannelNotFoundError; static PyObject *ChannelClosedError; static PyObject *ChannelEmptyError; static PyObject *ChannelNotEmptyError; +static PyObject *ChannelReleasedError; +static PyObject *NotReceivedError; static int channel_exceptions_init(PyObject *ns) @@ -322,6 +324,27 @@ channel_exceptions_init(PyObject *ns) return -1; } + // An operation tried to use a released channel. + ChannelReleasedError = PyErr_NewException( + "_interpreters.ChannelReleasedError", ChannelClosedError, NULL); + if (ChannelReleasedError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "ChannelReleasedError", ChannelReleasedError) != 0) { + return -1; + } + + // An operation trying to send an object when Nothing was waiting + // to receive it + NotReceivedError = PyErr_NewException( + "_interpreters.NotReceivedError", ChannelError, NULL); + if (NotReceivedError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "NotReceivedError", NotReceivedError) != 0) { + return -1; + } + // An operation tried to pop from an empty channel. ChannelEmptyError = PyErr_NewException( "_interpreters.ChannelEmptyError", ChannelError, NULL); @@ -484,6 +507,7 @@ typedef struct _channelend { struct _channelend *next; int64_t interp; int open; + int release; } _channelend; static _channelend * @@ -618,6 +642,10 @@ _channelends_associate(_channelends *ends, int64_t interp, int send) PyErr_SetString(ChannelClosedError, "channel already closed"); return -1; } + if (end->release && !end->open) { + PyErr_SetString(ChannelReleasedError, "channel released"); + return -1; + } // already associated return 0; } @@ -732,6 +760,7 @@ typedef struct _channel { _channelqueue *queue; _channelends *ends; int open; + int release; struct _channel_closing *closing; } _PyChannelState; @@ -789,6 +818,10 @@ _channel_add(_PyChannelState *chan, int64_t interp, PyErr_SetString(ChannelClosedError, "channel closed"); goto done; } + if (chan->release && !chan->open) { + PyErr_SetString(ChannelReleasedError, "channel released"); + return -1; + } if (_channelends_associate(chan->ends, interp, 1) != 0) { goto done; } @@ -813,6 +846,10 @@ _channel_next(_PyChannelState *chan, int64_t interp) PyErr_SetString(ChannelClosedError, "channel closed"); goto done; } + if (chan->release && !chan->open) { + PyErr_SetString(ChannelReleasedError, "channel released"); + goto done; + } if (_channelends_associate(chan->ends, interp, 0) != 0) { goto done; } @@ -1306,21 +1343,24 @@ _channel_destroy(_channels *channels, int64_t id) static int _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) { - const char *s = NULL; - Py_buffer view = {NULL, NULL}; - if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0){ + Py_buffer buffer; + PyObject *bytes; + + if (PyObject_GetBuffer(obj, &buffer, PyBUF_SIMPLE) < 0) { + PyErr_Format(PyExc_TypeError, + "Error creating object buffer, %.80s found", + Py_TYPE(obj)->tp_name); return -1; } - s = view.buf; - if (s == NULL) { - PyBuffer_Release(&view); + if (buffer.len == 0) { + PyBuffer_Release(&buffer); return -1; } PyInterpreterState *interp = _get_current(); if (interp == NULL) { - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } @@ -1328,7 +1368,7 @@ _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) PyThread_type_lock mutex = NULL; _PyChannelState *chan = _channels_lookup(channels, id, &mutex); if (chan == NULL) { - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } // Past this point we are responsible for releasing the mutex. @@ -1336,7 +1376,7 @@ _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) if (chan->closing != NULL) { PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", id); PyThread_release_lock(mutex); - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } @@ -1344,13 +1384,21 @@ _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); if (data == NULL) { PyThread_release_lock(mutex); - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } - if (_PyObject_GetCrossInterpreterData((PyObject *)s, data) != 0) { + + if (buffer.buf != NULL) + bytes = PyBytes_FromStringAndSize(buffer.buf, buffer.len); + else { + Py_INCREF(Py_None); + bytes = Py_None; + } + + if (_PyObject_GetCrossInterpreterData(bytes, data) != 0) { PyThread_release_lock(mutex); PyMem_Free(data); - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } @@ -1360,11 +1408,12 @@ _channel_send_buffer(_channels *channels, int64_t id, PyObject *obj) if (res != 0) { _PyCrossInterpreterData_Release(data); PyMem_Free(data); - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); return -1; } - PyBuffer_Release(&view); + PyBuffer_Release(&buffer); + return 0; } @@ -1467,6 +1516,13 @@ _channel_drop(_channels *channels, int64_t id, int send, int recv) } // Past this point we are responsible for releasing the mutex. + // Release the channel + if (!chan->release) { + PyErr_SetString(ChannelClosedError, "channel already released"); + return -1; + } + chan->release = 1; + // Close one or both of the two ends. int res = _channel_close_interpreter(chan, PyInterpreterState_GetID(interp), send-recv); PyThread_release_lock(mutex); From 9e2f9d6d6986a7628087064fdbbe8d25d29fc297 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2020 22:09:07 +0000 Subject: [PATCH 18/20] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst b/Misc/NEWS.d/next/Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst new file mode 100644 index 00000000000000..96cf52e5fc191b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-03-06-22-09-03.bpo-39881.Wh2TTV.rst @@ -0,0 +1,2 @@ +High-level Implementation of PEP 554. +(Patch By Joannah Nanjekye) \ No newline at end of file From 71ad4d7e5c116e004843cefec1ee03a29686a94c Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Thu, 7 May 2020 18:01:36 -0300 Subject: [PATCH 19/20] Update Doc/library/interpreters.rst Co-authored-by: Kyle Stanley --- Doc/library/interpreters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index 6b9c35b6250b89..df314e019dce28 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -8,7 +8,7 @@ -------------- -This module provides highlevel tools for working with sub-interpreters, +This module provides high-level tools for working with sub-interpreters, such as creating them, running code in them, or sending data between them. It is a wrapper around the low-level :mod:`_interpreters` module. From b12e3d2bca61955895269ac0ef6b109d1c3e0b84 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Thu, 7 May 2020 18:02:09 -0300 Subject: [PATCH 20/20] Update Doc/library/interpreters.rst Co-authored-by: Kyle Stanley --- Doc/library/interpreters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst index df314e019dce28..4c12f343033a4a 100644 --- a/Doc/library/interpreters.rst +++ b/Doc/library/interpreters.rst @@ -10,7 +10,7 @@ This module provides high-level tools for working with sub-interpreters, such as creating them, running code in them, or sending data between them. -It is a wrapper around the low-level :mod:`_interpreters` module. +It is a wrapper around the low-level `_interpreters` module. .. versionchanged:: added in 3.9 pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy