From 71ae2125197bcc791c7d6e7b45c097dd4bd81618 Mon Sep 17 00:00:00 2001 From: Bast0006 Date: Sat, 29 Mar 2025 07:22:50 -0700 Subject: [PATCH 1/8] GH-91153: Handle _getbytevalue potentially invalidating bytearray allocation during item assignment --- Objects/bytearrayobject.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 8f2d2dd02151c1..cd0c66ce036392 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -709,7 +709,8 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - char *buf = PyByteArray_AS_STRING(self); + // GH-91153: we cannot store a reference to the internal buffer here, as _getbytevalue might call into python code + // that could then invalidate it. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -744,7 +745,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value } else { assert(0 <= ival && ival < 256); - buf[i] = (char)ival; + PyByteArray_AS_STRING(self)[i] = (char)ival; return 0; } } @@ -805,6 +806,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Delete slice */ size_t cur; Py_ssize_t i; + char* buf = PyByteArray_AS_STRING(self); if (!_canresize(self)) return -1; @@ -845,6 +847,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Assign slice */ Py_ssize_t i; size_t cur; + char* buf = PyByteArray_AS_STRING(self); if (needed != slicelen) { PyErr_Format(PyExc_ValueError, From b2f7df50850f78e3f6bdbe6dbdb7f93f71661acf Mon Sep 17 00:00:00 2001 From: Bast Date: Thu, 10 Apr 2025 13:14:34 -0700 Subject: [PATCH 2/8] GH-91153: Add additional cases to `test_mutating_index` --- Lib/test/test_bytes.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 44486760c08349..8d2543df2883d4 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1889,6 +1889,41 @@ def __index__(self): with self.assertRaises(IndexError): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) + class BoomContinued: + def __index__(self): + nonlocal new_ba + # Clear the original bytearray, mutating it during index assignment. + # If the internal buffers are held over this operation, they become dangling + # However, this will fail a bounds check as above (as the clear sets bounds to zero) + ba.clear() + # At this moment, the bytearray potentially has a dangling pointer + # Create a new bytearray to catch any writes + new_ba = bytearray(0x180) + # Ensure bounds check passes + ba.extend([0] * 0x180) + return 0 + + with self.subTest("skip_bounds_safety"): + new_ba: bytearray + ba = bytearray(0x180) + ba[BoomContinued()] = ord("?") + self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_capi"): + new_ba: bytearray + ba = bytearray(0x180) + self._testlimitedcapi.sequence_setitem(ba, BoomContinued(), ord("?")) + self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_slice"): + new_ba: bytearray + ba = bytearray(0x180) + ba[BoomContinued():1] = [ord("?")] + self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + class AssortedBytesTest(unittest.TestCase): # From bb96c2a57ea7132c170b306c84bc9bbbb0307610 Mon Sep 17 00:00:00 2001 From: Bast Date: Sat, 17 May 2025 17:50:06 -0500 Subject: [PATCH 3/8] GH-91153: Move mutating index bytearray test to it's own separate test --- Lib/test/test_bytes.py | 43 ++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 8d2543df2883d4..cfa21bf8783b2d 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1889,40 +1889,43 @@ def __index__(self): with self.assertRaises(IndexError): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) - class BoomContinued: + def test_mutating_index_inbounds(self): + class MutatesOnIndex: + new_ba: bytearray + + def __init__(self): + self.ba = bytearray(0x180) + def __index__(self): - nonlocal new_ba # Clear the original bytearray, mutating it during index assignment. # If the internal buffers are held over this operation, they become dangling # However, this will fail a bounds check as above (as the clear sets bounds to zero) - ba.clear() + self.ba.clear() # At this moment, the bytearray potentially has a dangling pointer # Create a new bytearray to catch any writes - new_ba = bytearray(0x180) + self.new_ba = bytearray(0x180) # Ensure bounds check passes - ba.extend([0] * 0x180) + self.ba.extend([0] * 0x180) return 0 with self.subTest("skip_bounds_safety"): - new_ba: bytearray - ba = bytearray(0x180) - ba[BoomContinued()] = ord("?") - self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") - self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") with self.subTest("skip_bounds_safety_capi"): - new_ba: bytearray - ba = bytearray(0x180) - self._testlimitedcapi.sequence_setitem(ba, BoomContinued(), ord("?")) - self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") - self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self._testlimitedcapi.sequence_setitem(instance.ba, instance, ord("?")) + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") with self.subTest("skip_bounds_safety_slice"): - new_ba: bytearray - ba = bytearray(0x180) - ba[BoomContinued():1] = [ord("?")] - self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") - self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + instance = MutatesOnIndex() + instance.ba[instance:1] = [ord("?")] + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") class AssortedBytesTest(unittest.TestCase): From d29422b98873355cbf601e772a95f3ff516c062c Mon Sep 17 00:00:00 2001 From: Bast Date: Sat, 17 May 2025 18:22:52 -0500 Subject: [PATCH 4/8] GH-91153: add news entry --- .../2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst new file mode 100644 index 00000000000000..b337935fd3bb75 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst @@ -0,0 +1,3 @@ +Fix an issue where a :class:`bytearray` item assignment could crash or write +to the wrong bytearray when resized by the new value's :meth:`__index__` +method. From d3d19747e5257f4d8406270361951ec57a89fa48 Mon Sep 17 00:00:00 2001 From: Bast Date: Sat, 17 May 2025 20:56:56 -0500 Subject: [PATCH 5/8] Nit cleanup --- Lib/test/test_bytes.py | 2 ++ .../2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst | 3 --- .../2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst | 3 +++ 3 files changed, 5 insertions(+), 3 deletions(-) delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index cfa21bf8783b2d..c2843e726dfdcc 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1890,6 +1890,8 @@ def __index__(self): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) def test_mutating_index_inbounds(self): + # See gh-91153 + class MutatesOnIndex: new_ba: bytearray diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst deleted file mode 100644 index b337935fd3bb75..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue where a :class:`bytearray` item assignment could crash or write -to the wrong bytearray when resized by the new value's :meth:`__index__` -method. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst new file mode 100644 index 00000000000000..d90f7231355500 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst @@ -0,0 +1,3 @@ +Fix an issue where a :class:`bytearray` item assignment could crash or write +to the wrong bytearray when resized by the new value's +:meth:`~object.__index__` method. From 18e6597b9eac8d8f0e9ef1bbd617a68e0bd777b3 Mon Sep 17 00:00:00 2001 From: Bast <52266665+bast0006@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:44:52 -0700 Subject: [PATCH 6/8] GH-91153: Fixup comments and NEWS entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_bytes.py | 11 ++--------- .../2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst | 4 +--- Objects/bytearrayobject.c | 5 +++-- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index c2843e726dfdcc..f356cea2b1e659 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1893,21 +1893,14 @@ def test_mutating_index_inbounds(self): # See gh-91153 class MutatesOnIndex: - new_ba: bytearray def __init__(self): self.ba = bytearray(0x180) def __index__(self): - # Clear the original bytearray, mutating it during index assignment. - # If the internal buffers are held over this operation, they become dangling - # However, this will fail a bounds check as above (as the clear sets bounds to zero) self.ba.clear() - # At this moment, the bytearray potentially has a dangling pointer - # Create a new bytearray to catch any writes - self.new_ba = bytearray(0x180) - # Ensure bounds check passes - self.ba.extend([0] * 0x180) + self.new_ba = bytearray(0x180) # to catch out-of-bounds writes + self.ba.extend([0] * 0x180) # to check bounds checks return 0 with self.subTest("skip_bounds_safety"): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst index d90f7231355500..dc2f1e22ba5b05 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst @@ -1,3 +1 @@ -Fix an issue where a :class:`bytearray` item assignment could crash or write -to the wrong bytearray when resized by the new value's -:meth:`~object.__index__` method. +Fix a crash when a :class:`bytearray` is concurrently mutated during item assignment. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index cd0c66ce036392..38ffcb2ccb79a5 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -709,8 +709,9 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - // GH-91153: we cannot store a reference to the internal buffer here, as _getbytevalue might call into python code - // that could then invalidate it. + // Do not store a reference to the internal buffer since + // index.__index__() or _getbytevalue() may alter 'self'. + // See https://github.com/python/cpython/issues/91153. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); From 4b7ec9cb909770251e28ecf851dbad5a6b2fa351 Mon Sep 17 00:00:00 2001 From: Bast <52266665+bast0006@users.noreply.github.com> Date: Tue, 3 Jun 2025 13:50:26 -0700 Subject: [PATCH 7/8] GH-91153: Add short clarifying comments to test --- Lib/test/test_bytes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index f356cea2b1e659..fcc88682a3cbce 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1872,6 +1872,8 @@ def test_repeat_after_setslice(self): self.assertEqual(b3, b'xcxcxc') def test_mutating_index(self): + # bytearray slice assignment can call into python code + # that reallocates the internal buffer # See gh-91153 class Boom: @@ -1890,10 +1892,10 @@ def __index__(self): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) def test_mutating_index_inbounds(self): - # See gh-91153 + # gh-91153 continued + # Ensure buffer is not broken even if length is correct class MutatesOnIndex: - def __init__(self): self.ba = bytearray(0x180) From 970c1ad51e85523917e898e147dcce738e161157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 12 Jul 2025 15:10:54 +0200 Subject: [PATCH 8/8] Apply suggestions from code review --- Objects/bytearrayobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 001c345ade6024..bf30c06af5d8fa 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -807,7 +807,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Delete slice */ size_t cur; Py_ssize_t i; - char* buf = PyByteArray_AS_STRING(self); + char *buf = PyByteArray_AS_STRING(self); if (!_canresize(self)) return -1; @@ -848,7 +848,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Assign slice */ Py_ssize_t i; size_t cur; - char* buf = PyByteArray_AS_STRING(self); + char *buf = PyByteArray_AS_STRING(self); if (needed != slicelen) { PyErr_Format(PyExc_ValueError, pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy