Skip to content

Commit 934d648

Browse files
ericsnowcurrentlyGlyphack
ericsnowcurrently
authored and
Glyphack
committed
pythongh-84570: Send-Wait Fixes for _xxinterpchannels (pythongh-111006)
There were a few things I did in pythongh-110565 that need to be fixed. I also forgot to add tests in that PR. (Note that this PR exposes a refleak introduced by pythongh-110246. I'll take care of that separately.)
1 parent e508630 commit 934d648

File tree

5 files changed

+571
-148
lines changed

5 files changed

+571
-148
lines changed

‎Include/internal/pycore_pythread.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,21 @@ extern int _PyThread_at_fork_reinit(PyThread_type_lock *lock);
8686
#endif /* HAVE_FORK */
8787

8888

89+
// unset: -1 seconds, in nanoseconds
90+
#define PyThread_UNSET_TIMEOUT ((_PyTime_t)(-1 * 1000 * 1000 * 1000))
91+
92+
/* Helper to acquire an interruptible lock with a timeout. If the lock acquire
93+
* is interrupted, signal handlers are run, and if they raise an exception,
94+
* PY_LOCK_INTR is returned. Otherwise, PY_LOCK_ACQUIRED or PY_LOCK_FAILURE
95+
* are returned, depending on whether the lock can be acquired within the
96+
* timeout.
97+
*/
98+
// Exported for the _xxinterpchannels module.
99+
PyAPI_FUNC(PyLockStatus) PyThread_acquire_lock_timed_with_retries(
100+
PyThread_type_lock,
101+
PY_TIMEOUT_T microseconds);
102+
103+
89104
#ifdef __cplusplus
90105
}
91106
#endif

‎Lib/test/test__xxinterpchannels.py

Lines changed: 173 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -564,7 +564,62 @@ def test_channel_list_interpreters_closed_send_end(self):
564564
with self.assertRaises(channels.ChannelClosedError):
565565
channels.list_interpreters(cid, send=False)
566566

567-
####################
567+
def test_allowed_types(self):
568+
cid = channels.create()
569+
objects = [
570+
None,
571+
'spam',
572+
b'spam',
573+
42,
574+
]
575+
for obj in objects:
576+
with self.subTest(obj):
577+
channels.send(cid, obj, blocking=False)
578+
got = channels.recv(cid)
579+
580+
self.assertEqual(got, obj)
581+
self.assertIs(type(got), type(obj))
582+
# XXX Check the following?
583+
#self.assertIsNot(got, obj)
584+
# XXX What about between interpreters?
585+
586+
def test_run_string_arg_unresolved(self):
587+
cid = channels.create()
588+
interp = interpreters.create()
589+
590+
out = _run_output(interp, dedent("""
591+
import _xxinterpchannels as _channels
592+
print(cid.end)
593+
_channels.send(cid, b'spam', blocking=False)
594+
"""),
595+
dict(cid=cid.send))
596+
obj = channels.recv(cid)
597+
598+
self.assertEqual(obj, b'spam')
599+
self.assertEqual(out.strip(), 'send')
600+
601+
# XXX For now there is no high-level channel into which the
602+
# sent channel ID can be converted...
603+
# Note: this test caused crashes on some buildbots (bpo-33615).
604+
@unittest.skip('disabled until high-level channels exist')
605+
def test_run_string_arg_resolved(self):
606+
cid = channels.create()
607+
cid = channels._channel_id(cid, _resolve=True)
608+
interp = interpreters.create()
609+
610+
out = _run_output(interp, dedent("""
611+
import _xxinterpchannels as _channels
612+
print(chan.id.end)
613+
_channels.send(chan.id, b'spam', blocking=False)
614+
"""),
615+
dict(chan=cid.send))
616+
obj = channels.recv(cid)
617+
618+
self.assertEqual(obj, b'spam')
619+
self.assertEqual(out.strip(), 'send')
620+
621+
#-------------------
622+
# send/recv
568623

569624
def test_send_recv_main(self):
570625
cid = channels.create()
@@ -705,6 +760,9 @@ def test_recv_sending_interp_destroyed(self):
705760
channels.recv(cid2)
706761
del cid2
707762

763+
#-------------------
764+
# send_buffer
765+
708766
def test_send_buffer(self):
709767
buf = bytearray(b'spamspamspam')
710768
cid = channels.create()
@@ -720,60 +778,131 @@ def test_send_buffer(self):
720778
obj[4:8] = b'ham.'
721779
self.assertEqual(obj, buf)
722780

723-
def test_allowed_types(self):
781+
#-------------------
782+
# send with waiting
783+
784+
def build_send_waiter(self, obj, *, buffer=False):
785+
# We want a long enough sleep that send() actually has to wait.
786+
787+
if buffer:
788+
send = channels.send_buffer
789+
else:
790+
send = channels.send
791+
724792
cid = channels.create()
725-
objects = [
726-
None,
727-
'spam',
728-
b'spam',
729-
42,
730-
]
731-
for obj in objects:
732-
with self.subTest(obj):
733-
channels.send(cid, obj, blocking=False)
734-
got = channels.recv(cid)
793+
try:
794+
started = time.monotonic()
795+
send(cid, obj, blocking=False)
796+
stopped = time.monotonic()
797+
channels.recv(cid)
798+
finally:
799+
channels.destroy(cid)
800+
delay = stopped - started # seconds
801+
delay *= 3
735802

736-
self.assertEqual(got, obj)
737-
self.assertIs(type(got), type(obj))
738-
# XXX Check the following?
739-
#self.assertIsNot(got, obj)
740-
# XXX What about between interpreters?
803+
def wait():
804+
time.sleep(delay)
805+
return wait
741806

742-
def test_run_string_arg_unresolved(self):
807+
def test_send_blocking_waiting(self):
808+
received = None
809+
obj = b'spam'
810+
wait = self.build_send_waiter(obj)
743811
cid = channels.create()
744-
interp = interpreters.create()
812+
def f():
813+
nonlocal received
814+
wait()
815+
received = recv_wait(cid)
816+
t = threading.Thread(target=f)
817+
t.start()
818+
channels.send(cid, obj, blocking=True)
819+
t.join()
745820

746-
out = _run_output(interp, dedent("""
747-
import _xxinterpchannels as _channels
748-
print(cid.end)
749-
_channels.send(cid, b'spam', blocking=False)
750-
"""),
751-
dict(cid=cid.send))
752-
obj = channels.recv(cid)
821+
self.assertEqual(received, obj)
753822

754-
self.assertEqual(obj, b'spam')
755-
self.assertEqual(out.strip(), 'send')
823+
def test_send_buffer_blocking_waiting(self):
824+
received = None
825+
obj = bytearray(b'spam')
826+
wait = self.build_send_waiter(obj, buffer=True)
827+
cid = channels.create()
828+
def f():
829+
nonlocal received
830+
wait()
831+
received = recv_wait(cid)
832+
t = threading.Thread(target=f)
833+
t.start()
834+
channels.send_buffer(cid, obj, blocking=True)
835+
t.join()
756836

757-
# XXX For now there is no high-level channel into which the
758-
# sent channel ID can be converted...
759-
# Note: this test caused crashes on some buildbots (bpo-33615).
760-
@unittest.skip('disabled until high-level channels exist')
761-
def test_run_string_arg_resolved(self):
837+
self.assertEqual(received, obj)
838+
839+
def test_send_blocking_no_wait(self):
840+
received = None
841+
obj = b'spam'
762842
cid = channels.create()
763-
cid = channels._channel_id(cid, _resolve=True)
764-
interp = interpreters.create()
843+
def f():
844+
nonlocal received
845+
received = recv_wait(cid)
846+
t = threading.Thread(target=f)
847+
t.start()
848+
channels.send(cid, obj, blocking=True)
849+
t.join()
765850

766-
out = _run_output(interp, dedent("""
767-
import _xxinterpchannels as _channels
768-
print(chan.id.end)
769-
_channels.send(chan.id, b'spam', blocking=False)
770-
"""),
771-
dict(chan=cid.send))
772-
obj = channels.recv(cid)
851+
self.assertEqual(received, obj)
773852

774-
self.assertEqual(obj, b'spam')
775-
self.assertEqual(out.strip(), 'send')
853+
def test_send_buffer_blocking_no_wait(self):
854+
received = None
855+
obj = bytearray(b'spam')
856+
cid = channels.create()
857+
def f():
858+
nonlocal received
859+
received = recv_wait(cid)
860+
t = threading.Thread(target=f)
861+
t.start()
862+
channels.send_buffer(cid, obj, blocking=True)
863+
t.join()
864+
865+
self.assertEqual(received, obj)
866+
867+
def test_send_closed_while_waiting(self):
868+
obj = b'spam'
869+
wait = self.build_send_waiter(obj)
870+
cid = channels.create()
871+
def f():
872+
wait()
873+
channels.close(cid, force=True)
874+
t = threading.Thread(target=f)
875+
t.start()
876+
with self.assertRaises(channels.ChannelClosedError):
877+
channels.send(cid, obj, blocking=True)
878+
t.join()
879+
880+
def test_send_buffer_closed_while_waiting(self):
881+
try:
882+
self._has_run_once
883+
except AttributeError:
884+
# At the moment, this test leaks a few references.
885+
# It looks like the leak originates with the addition
886+
# of _channels.send_buffer() (gh-110246), whereas the
887+
# tests were added afterward. We want this test even
888+
# if the refleak isn't fixed yet, so we skip here.
889+
raise unittest.SkipTest('temporarily skipped due to refleaks')
890+
else:
891+
self._has_run_once = True
892+
893+
obj = bytearray(b'spam')
894+
wait = self.build_send_waiter(obj, buffer=True)
895+
cid = channels.create()
896+
def f():
897+
wait()
898+
channels.close(cid, force=True)
899+
t = threading.Thread(target=f)
900+
t.start()
901+
with self.assertRaises(channels.ChannelClosedError):
902+
channels.send_buffer(cid, obj, blocking=True)
903+
t.join()
776904

905+
#-------------------
777906
# close
778907

779908
def test_close_single_user(self):

‎Modules/_threadmodule.c

Lines changed: 2 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
/* Interface to Sjoerd's portable C thread library */
44

55
#include "Python.h"
6-
#include "pycore_ceval.h" // _PyEval_MakePendingCalls()
76
#include "pycore_dict.h" // _PyDict_Pop()
87
#include "pycore_interp.h" // _PyInterpreterState.threads.count
98
#include "pycore_moduleobject.h" // _PyModule_GetState()
@@ -76,57 +75,10 @@ lock_dealloc(lockobject *self)
7675
Py_DECREF(tp);
7776
}
7877

79-
/* Helper to acquire an interruptible lock with a timeout. If the lock acquire
80-
* is interrupted, signal handlers are run, and if they raise an exception,
81-
* PY_LOCK_INTR is returned. Otherwise, PY_LOCK_ACQUIRED or PY_LOCK_FAILURE
82-
* are returned, depending on whether the lock can be acquired within the
83-
* timeout.
84-
*/
85-
static PyLockStatus
78+
static inline PyLockStatus
8679
acquire_timed(PyThread_type_lock lock, _PyTime_t timeout)
8780
{
88-
PyThreadState *tstate = _PyThreadState_GET();
89-
_PyTime_t endtime = 0;
90-
if (timeout > 0) {
91-
endtime = _PyDeadline_Init(timeout);
92-
}
93-
94-
PyLockStatus r;
95-
do {
96-
_PyTime_t microseconds;
97-
microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING);
98-
99-
/* first a simple non-blocking try without releasing the GIL */
100-
r = PyThread_acquire_lock_timed(lock, 0, 0);
101-
if (r == PY_LOCK_FAILURE && microseconds != 0) {
102-
Py_BEGIN_ALLOW_THREADS
103-
r = PyThread_acquire_lock_timed(lock, microseconds, 1);
104-
Py_END_ALLOW_THREADS
105-
}
106-
107-
if (r == PY_LOCK_INTR) {
108-
/* Run signal handlers if we were interrupted. Propagate
109-
* exceptions from signal handlers, such as KeyboardInterrupt, by
110-
* passing up PY_LOCK_INTR. */
111-
if (_PyEval_MakePendingCalls(tstate) < 0) {
112-
return PY_LOCK_INTR;
113-
}
114-
115-
/* If we're using a timeout, recompute the timeout after processing
116-
* signals, since those can take time. */
117-
if (timeout > 0) {
118-
timeout = _PyDeadline_Get(endtime);
119-
120-
/* Check for negative values, since those mean block forever.
121-
*/
122-
if (timeout < 0) {
123-
r = PY_LOCK_FAILURE;
124-
}
125-
}
126-
}
127-
} while (r == PY_LOCK_INTR); /* Retry if we were interrupted. */
128-
129-
return r;
81+
return PyThread_acquire_lock_timed_with_retries(lock, timeout);
13082
}
13183

13284
static int

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy