Skip to content

gh-76785: Expand How Interpreter Queues Handle Interpreter Finalization #116431

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Do not convert the unbound op between str and int.
  • Loading branch information
ericsnowcurrently committed Mar 6, 2024
commit cc5354d015f6a9017ff3331b753c6130e3b787ba
76 changes: 44 additions & 32 deletions Lib/test/support/interpreters/queues.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,28 +65,37 @@ def __repr__(self):
UNBOUND_ERROR = object()
UNBOUND_REMOVE = object()

_UNBOUND_CONSTANT_TO_FLAG = {
UNBOUND_REMOVE: 1,
UNBOUND_ERROR: 2,
UNBOUND: 3,
}
_UNBOUND_FLAG_TO_CONSTANT = {v: k
for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()}

def _serialize_unbound(unbound):
if unbound is UNBOUND_REMOVE:
unbound = 'remove'
elif unbound is UNBOUND_ERROR:
unbound = 'error'
elif unbound is UNBOUND:
unbound = 'replace'
else:
raise NotImplementedError(f'unsupported unbound replacement {unbound !r}')
return unbound


def _resolve_unbound(unbound):
if unbound == 'remove':
raise RuntimeError('"remove" not possible here')
elif unbound == 'error':
op = unbound
try:
flag = _UNBOUND_CONSTANT_TO_FLAG[op]
except KeyError:
raise NotImplementedError(f'unsupported unbound replacement op {op!r}')
return flag,


def _resolve_unbound(flag):
try:
op = _UNBOUND_FLAG_TO_CONSTANT[flag]
except KeyError:
raise NotImplementedError(f'unsupported unbound replacement op {flag!r}')
if op is UNBOUND_REMOVE:
# "remove" not possible here
raise NotImplementedError
elif op is UNBOUND_ERROR:
raise ItemInterpreterDestroyed("item's original interpreter destroyed")
elif unbound == 'replace':
elif op is UNBOUND:
return UNBOUND
else:
raise NotImplementedError(f'{unbound!r} unsupported')
raise NotImplementedError(repr(op))


def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND):
Expand All @@ -103,7 +112,8 @@ def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND):
"""
fmt = _SHARED_ONLY if syncobj else _PICKLED
unbound = _serialize_unbound(unbounditems)
qid = _queues.create(maxsize, fmt, unbound)
unboundop, = unbound
qid = _queues.create(maxsize, fmt, unboundop)
return Queue(qid, _fmt=fmt, _unbound=unbound)


Expand All @@ -126,11 +136,13 @@ def __new__(cls, id, /, *, _fmt=None, _unbound=None):
raise TypeError(f'id must be an int, got {id!r}')
if _fmt is None:
if _unbound is None:
_fmt, _unbound = _queues.get_queue_defaults(id)
_fmt, op = _queues.get_queue_defaults(id)
_unbound = (op,)
else:
_fmt, _ = _queues.get_queue_defaults(id)
elif _unbound is None:
_, _unbound = _queues.get_queue_defaults(id)
_, op = _queues.get_queue_defaults(id)
_unbound = (op,)
try:
self = _known_queues[id]
except KeyError:
Expand Down Expand Up @@ -241,9 +253,9 @@ def put(self, obj, timeout=None, *,
else:
fmt = _SHARED_ONLY if syncobj else _PICKLED
if unbound is None:
unbound = self._unbound
unboundop, = self._unbound
else:
unbound = _serialize_unbound(unbound)
unboundop, = _serialize_unbound(unbound)
if timeout is not None:
timeout = int(timeout)
if timeout < 0:
Expand All @@ -253,7 +265,7 @@ def put(self, obj, timeout=None, *,
obj = pickle.dumps(obj)
while True:
try:
_queues.put(self._id, obj, fmt, unbound)
_queues.put(self._id, obj, fmt, unboundop)
except QueueFull as exc:
if timeout is not None and time.time() >= end:
raise # re-raise
Expand All @@ -267,12 +279,12 @@ def put_nowait(self, obj, *, syncobj=None, unbound=None):
else:
fmt = _SHARED_ONLY if syncobj else _PICKLED
if unbound is None:
unbound = self._unbound
unboundop, = self._unbound
else:
unbound = _serialize_unbound(unbound)
unboundop, = _serialize_unbound(unbound)
if fmt is _PICKLED:
obj = pickle.dumps(obj)
_queues.put(self._id, obj, fmt, unbound)
_queues.put(self._id, obj, fmt, unboundop)

def get(self, timeout=None, *,
_delay=10 / 1000, # 10 milliseconds
Expand All @@ -292,16 +304,16 @@ def get(self, timeout=None, *,
end = time.time() + timeout
while True:
try:
obj, fmt, unbound = _queues.get(self._id)
obj, fmt, unboundop = _queues.get(self._id)
except QueueEmpty as exc:
if timeout is not None and time.time() >= end:
raise # re-raise
time.sleep(_delay)
else:
break
if unbound is not None:
if unboundop is not None:
assert obj is None, repr(obj)
return _resolve_unbound(unbound)
return _resolve_unbound(unboundop)
if fmt == _PICKLED:
obj = pickle.loads(obj)
else:
Expand All @@ -315,12 +327,12 @@ def get_nowait(self):
is the same as get().
"""
try:
obj, fmt, unbound = _queues.get(self._id)
obj, fmt, unboundop = _queues.get(self._id)
except QueueEmpty as exc:
raise # re-raise
if unbound is not None:
if unboundop is not None:
assert obj is None, repr(obj)
return _resolve_unbound(unbound)
return _resolve_unbound(unboundop)
if fmt == _PICKLED:
obj = pickle.loads(obj)
else:
Expand Down
15 changes: 9 additions & 6 deletions Lib/test/test_interpreters/test_queues.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
from .utils import _run_output, TestBase as _TestBase


REPLACE = queues._UNBOUND_CONSTANT_TO_FLAG[queues.UNBOUND]


def get_num_queues():
return len(_queues.list_all())

Expand Down Expand Up @@ -40,7 +43,7 @@ def test_highlevel_reloaded(self):
importlib.reload(queues)

def test_create_destroy(self):
qid = _queues.create(2, 0, 'replace')
qid = _queues.create(2, 0, REPLACE)
_queues.destroy(qid)
self.assertEqual(get_num_queues(), 0)
with self.assertRaises(queues.QueueNotFoundError):
Expand All @@ -54,7 +57,7 @@ def test_not_destroyed(self):
'-c',
dedent(f"""
import {_queues.__name__} as _queues
_queues.create(2, 0, 'replace')
_queues.create(2, 0, {REPLACE})
"""),
)
self.assertEqual(stdout, '')
Expand All @@ -65,29 +68,29 @@ def test_not_destroyed(self):

def test_bind_release(self):
with self.subTest('typical'):
qid = _queues.create(2, 0, 'replace')
qid = _queues.create(2, 0, REPLACE)
_queues.bind(qid)
_queues.release(qid)
self.assertEqual(get_num_queues(), 0)

with self.subTest('bind too much'):
qid = _queues.create(2, 0, 'replace')
qid = _queues.create(2, 0, REPLACE)
_queues.bind(qid)
_queues.bind(qid)
_queues.release(qid)
_queues.destroy(qid)
self.assertEqual(get_num_queues(), 0)

with self.subTest('nested'):
qid = _queues.create(2, 0, 'replace')
qid = _queues.create(2, 0, REPLACE)
_queues.bind(qid)
_queues.bind(qid)
_queues.release(qid)
_queues.release(qid)
self.assertEqual(get_num_queues(), 0)

with self.subTest('release without binding'):
qid = _queues.create(2, 0, 'replace')
qid = _queues.create(2, 0, REPLACE)
with self.assertRaises(queues.QueueError):
_queues.release(qid)

Expand Down
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy