From f54c5fc6c0394ea322d82aed3e338544251ec7f9 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 1 Mar 2025 09:43:58 -0500 Subject: [PATCH 01/32] slow arraymodule --- Lib/test/test_array.py | 265 +++++++++++++++++ Modules/arraymodule.c | 520 +++++++++++++++++++++++---------- Modules/clinic/arraymodule.c.h | 156 +++++++++- 3 files changed, 782 insertions(+), 159 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 58ea89c4fac833..a7d2105b938598 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -3,16 +3,20 @@ """ import collections.abc +import io import unittest from test import support from test.support import import_helper from test.support import os_helper +from test.support import threading_helper from test.support import _2G import weakref import pickle import operator +import random import struct import sys +import threading import warnings import array @@ -1673,5 +1677,266 @@ def test_gh_128961(self): self.assertRaises(StopIteration, next, it) +class FreeThreadingTest(unittest.TestCase): + # Test pretty much everything that can break under free-threading. + # Non-deterministic, but at least one of these things will fail if + # array module is not free-thread safe. + + @unittest.skipUnless(support.Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_free_threading(self): + def pop1(b, a): # MODIFIES! + b.wait() + try: a.pop() + except IndexError: pass + + def append1(b, a): # MODIFIES! + b.wait() + a.append(2) + + def insert1(b, a): # MODIFIES! + b.wait() + a.insert(0, 2) + + def extend(b, a): # MODIFIES! + c = array.array('i', [2]) + b.wait() + a.extend(c) + + def extend2(b, a, c): # MODIFIES! + b.wait() + a.extend(c) + + def inplace_concat(b, a): # MODIFIES! + c = array.array('i', [2]) + b.wait() + a += c + + def inplace_concat2(b, a, c): # MODIFIES! + b.wait() + a += c + + def inplace_repeat2(b, a): # MODIFIES! + b.wait() + a *= 2 + + def clear(b, a, *args): # MODIFIES! + b.wait() + a.clear() + + def clear2(b, a, c): # MODIFIES c! + b.wait() + try: c.clear() + except BufferError: pass + + def remove1(b, a): # MODIFIES! + b.wait() + try: a.remove(1) + except ValueError: pass + + def fromunicode(b, a): # MODIFIES! + b.wait() + a.fromunicode('test') + + def frombytes(b, a): # MODIFIES! + b.wait() + a.frombytes(b'0000') + + def frombytes2(b, a, c): # MODIFIES! + b.wait() + a.frombytes(c) + + def fromlist(b, a): # MODIFIES! + n = random.randint(0, 100) + b.wait() + a.fromlist([2] * n) + + def ass_subscr2(b, a, c): # MODIFIES! + b.wait() + a[:] = c + + def ass0(b, a): # modifies inplace + b.wait() + try: a[0] = 0 + except IndexError: pass + + def byteswap(b, a): # modifies inplace + b.wait() + a.byteswap() + + def tounicode(b, a): + b.wait() + a.tounicode() + + def tobytes(b, a): + b.wait() + a.tobytes() + + def tolist(b, a): + b.wait() + a.tolist() + + def tofile(b, a): + f = io.BytesIO() + b.wait() + a.tofile(f) + + def reduce_ex2(b, a): + b.wait() + a.__reduce_ex__(2) + + def reduce_ex3(b, a): + b.wait() + c = a.__reduce_ex__(3) + assert not c[1] or 0xdd not in c[1][3] + + def copy(b, a): + b.wait() + c = a.__copy__() + assert not c or 0xdd not in c + + def repr1(b, a): + b.wait() + repr(a) + + def repeat2(b, a): + b.wait() + a * 2 + + def count1(b, a): + b.wait() + a.count(1) + + def index1(b, a): + b.wait() + try: a.index(1) + except ValueError: pass + + def contains1(b, a): + b.wait() + try: 1 in a + except ValueError: pass + + def subscr0(b, a): + b.wait() + try: a[0] + except IndexError: pass + + def concat(b, a): + b.wait() + a + a + + def concat2(b, a, c): + b.wait() + a + c + + def richcmplhs(b, a): + c = a[:] + b.wait() + a == c + + def richcmprhs(b, a): + c = a[:] + b.wait() + c == a + + def new(b, a): + tc = a.typecode + b.wait() + array.array(tc, a) + + def repr_(b, a): + b.wait() + repr(a) + + def irepeat(b, a): # MODIFIES! + b.wait() + a *= 2 + + def newi(b, l): + b.wait() + array.array('i', l) + + def fromlistl(b, a, l): # MODIFIES! + b.wait() + a.fromlist(l) + + def fromlistlclear(b, a, l): # MODIFIES LIST! + b.wait() + l.clear() + + def iter_next(b, a, it): # MODIFIES ITERATOR! + b.wait() + list(it) + + def iter_reduce(b, a, it): + b.wait() + c = it.__reduce__() + assert not c[1] or 0xdd not in c[1][0] + + def check(funcs, a=None, *args): + if a is None: + a = array.array('i', [1]) + + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier, a, *args)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + check([pop1] * 10) + check([pop1] + [subscr0] * 10) + check([append1] * 10) + check([insert1] * 10) + check([pop1] + [index1] * 10) + check([pop1] + [contains1] * 10) + check([insert1] + [repeat2] * 10) + check([pop1] + [repr1] * 10) + check([inplace_repeat2] * 10) + check([byteswap] * 10) + check([insert1] + [clear] * 10) + check([pop1] + [count1] * 10) + check([remove1] * 10) + check([clear] + [copy] * 10, array.array('B', b'0' * 0x400000)) + check([pop1] + [reduce_ex2] * 10) + check([clear] + [reduce_ex3] * 10, array.array('B', b'0' * 0x400000)) + check([pop1] + [tobytes] * 10) + check([pop1] + [tolist] * 10) + check([clear, tounicode] * 10, array.array('w', 'a'*10000)) + check([clear, tofile] * 10, array.array('w', 'a'*10000)) + check([clear] + [extend] * 10) + check([clear] + [inplace_concat] * 10) + check([clear] + [concat] * 10, array.array('w', 'a'*10000)) + check([fromunicode] * 10, array.array('w', 'a')) + check([frombytes] * 10) + check([fromlist] * 10) + check([clear] + [richcmplhs] * 10, array.array('i', [1]*10000)) + check([clear] + [richcmprhs] * 10, array.array('i', [1]*10000)) + check([clear, ass0] * 10, array.array('i', [1]*10000)) # to test array_ass_item must disable Py_mp_ass_subscript + check([clear] + [new] * 10, array.array('w', 'a'*10000)) + check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) + check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) + check([clear] + [irepeat] * 10, array.array('B', b'0' * 0x40000)) + check([clear] + [iter_reduce] * 10, a := array.array('B', b'0' * 0x400), iter(a)) + + # make sure we handle non-self objects correctly + check([clear] + [newi] * 10, [2] * random.randint(0, 100)) + check([fromlistlclear] + [fromlistl] * 10, array.array('i', [1]), [2] * random.randint(0, 100)) + check([clear2] + [concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + check([clear2] + [inplace_concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + check([clear2] + [extend2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + check([clear2] + [ass_subscr2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + check([clear2] + [frombytes2] * 10, array.array('w', 'a'*10000), array.array('B', b'a'*10000)) + + # iterator stuff + check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 5b86ec98393e48..0775b26e1d68ed 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -13,6 +13,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_pyatomic_ft_wrappers.h" #include // offsetof() #include @@ -68,6 +69,19 @@ typedef struct { PyObject *str_iter; } array_state; +static inline Py_ssize_t Pyarrayobject_GET_SIZE(PyObject *op) { + arrayobject *ao = (arrayobject *)op; +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(ao)->ob_size)); +#else + return Py_SIZE(ao); +#endif +} +#define Pyarrayobject_GET_SIZE(op) Pyarrayobject_GET_SIZE(_PyObject_CAST(op)) + +/* Forward declaration. */ +static PyObject *array_array_frombytes(PyObject *self, PyObject *bytes); + static array_state * get_array_state(PyObject *module) { @@ -133,6 +147,7 @@ enum machine_format_code { static int array_resize(arrayobject *self, Py_ssize_t newsize) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); char *items; size_t _new_size; @@ -158,7 +173,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) PyMem_Free(self->ob_item); self->ob_item = NULL; Py_SET_SIZE(self, 0); - self->allocated = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, 0); return 0; } @@ -188,7 +203,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } self->ob_item = items; Py_SET_SIZE(self, newsize); - self->allocated = _new_size; + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, _new_size); return 0; } @@ -672,6 +687,7 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des static PyObject * getarrayitem(PyObject *op, Py_ssize_t i) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(op)); assert(array_Check(op, state)); @@ -685,6 +701,7 @@ getarrayitem(PyObject *op, Py_ssize_t i) static int ins1(arrayobject *self, Py_ssize_t where, PyObject *v) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); char *items; Py_ssize_t n = Py_SIZE(self); if (v == NULL) { @@ -728,6 +745,11 @@ array_dealloc(PyObject *op) PyObject_GC_UnTrack(op); arrayobject *self = arrayobject_CAST(op); + if (self->ob_exports > 0) { + PyErr_SetString(PyExc_SystemError, + "deallocated array object has exported buffers"); + PyErr_Print(); + } if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(op); } @@ -739,8 +761,10 @@ array_dealloc(PyObject *op) } static PyObject * -array_richcompare(PyObject *v, PyObject *w, int op) +array_richcompare_lock_held(PyObject *v, PyObject *w, int op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(v); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(w); array_state *state = find_array_state_by_type(Py_TYPE(v)); arrayobject *va, *wa; PyObject *vi = NULL; @@ -848,15 +872,27 @@ array_richcompare(PyObject *v, PyObject *w, int op) return res; } +static PyObject * +array_richcompare(PyObject *v, PyObject *w, int op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(v, w); + ret = array_richcompare_lock_held(v, w, op); + Py_END_CRITICAL_SECTION2(); + return ret; +} + static Py_ssize_t array_length(PyObject *op) { - return Py_SIZE(op); + arrayobject *self = arrayobject_CAST(op); + return Pyarrayobject_GET_SIZE(self); } static PyObject * -array_item(PyObject *op, Py_ssize_t i) +array_item_lock_held(PyObject *op, Py_ssize_t i) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); if (i < 0 || i >= Py_SIZE(op)) { PyErr_SetString(PyExc_IndexError, "array index out of range"); return NULL; @@ -864,9 +900,20 @@ array_item(PyObject *op, Py_ssize_t i) return getarrayitem(op, i); } +static PyObject * +array_item(PyObject *op, Py_ssize_t i) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_item_lock_held(op, i); + Py_END_CRITICAL_SECTION(); + return ret; +} + static PyObject * array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a); array_state *state = find_array_state_by_type(Py_TYPE(a)); arrayobject *np; @@ -891,6 +938,7 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) } /*[clinic input] +@critical_section array.array.clear Remove all items from the array. @@ -898,7 +946,7 @@ Remove all items from the array. static PyObject * array_array_clear_impl(arrayobject *self) -/*[clinic end generated code: output=5efe0417062210a9 input=5dffa30e94e717a4]*/ +/*[clinic end generated code: output=5efe0417062210a9 input=1c9dfcc80f5b6731]*/ { if (array_resize(self, 0) == -1) { return NULL; @@ -907,6 +955,7 @@ array_array_clear_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.__copy__ Return a copy of the array. @@ -914,12 +963,13 @@ Return a copy of the array. static PyObject * array_array___copy___impl(arrayobject *self) -/*[clinic end generated code: output=dec7c3f925d9619e input=ad1ee5b086965f09]*/ +/*[clinic end generated code: output=dec7c3f925d9619e input=7622f8f9489472d5]*/ { return array_slice(self, 0, Py_SIZE(self)); } /*[clinic input] +@critical_section array.array.__deepcopy__ unused: object @@ -929,15 +979,17 @@ Return a copy of the array. [clinic start generated code]*/ static PyObject * -array_array___deepcopy__(arrayobject *self, PyObject *unused) -/*[clinic end generated code: output=1ec748d8e14a9faa input=2405ecb4933748c4]*/ +array_array___deepcopy___impl(arrayobject *self, PyObject *unused) +/*[clinic end generated code: output=703b4c412feaaf31 input=1a29f718f5b8a1dc]*/ { return array_array___copy___impl(self); } static PyObject * -array_concat(PyObject *op, PyObject *bb) +array_concat_lock_held(PyObject *op, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb); arrayobject *a = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(a)); Py_ssize_t size; @@ -973,8 +1025,19 @@ array_concat(PyObject *op, PyObject *bb) } static PyObject * -array_repeat(PyObject *op, Py_ssize_t n) +array_concat(PyObject *op, PyObject *bb) { + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(op, bb); + ret = array_concat_lock_held(op, bb); + Py_END_CRITICAL_SECTION2(); + return ret; +} + +static PyObject * +array_repeat_lock_held(PyObject *op, Py_ssize_t n) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *a = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(a)); @@ -998,9 +1061,20 @@ array_repeat(PyObject *op, Py_ssize_t n) return (PyObject *)np; } +static PyObject * +array_repeat(PyObject *op, Py_ssize_t n) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_repeat_lock_held(op, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a); char *item; Py_ssize_t d; /* Change in size */ if (ilow < 0) @@ -1034,8 +1108,9 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) } static int -array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) +setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *a = arrayobject_CAST(op); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, @@ -1048,18 +1123,20 @@ array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) } static int -setarrayitem(PyObject *a, Py_ssize_t i, PyObject *v) +array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) { -#ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(a)); - assert(array_Check(a, state)); -#endif - return array_ass_item(a, i, v); + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = setarrayitem(op, i, v); + Py_END_CRITICAL_SECTION(); + return ret; } static int array_iter_extend(arrayobject *self, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb); PyObject *it, *v; it = PyObject_GetIter(bb); @@ -1081,8 +1158,10 @@ array_iter_extend(arrayobject *self, PyObject *bb) } static int -array_do_extend(array_state *state, arrayobject *self, PyObject *bb) +array_do_extend_lock_held(array_state *state, arrayobject *self, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb); Py_ssize_t size, oldsize, bbsize; if (!array_Check(bb, state)) @@ -1113,6 +1192,16 @@ array_do_extend(array_state *state, arrayobject *self, PyObject *bb) #undef b } +static int +array_do_extend(array_state *state, arrayobject *self, PyObject *bb) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION2(self, bb); + ret = array_do_extend_lock_held(state, self, bb); + Py_END_CRITICAL_SECTION2(); + return ret; +} + static PyObject * array_inplace_concat(PyObject *op, PyObject *bb) { @@ -1131,8 +1220,9 @@ array_inplace_concat(PyObject *op, PyObject *bb) } static PyObject * -array_inplace_repeat(PyObject *op, Py_ssize_t n) +array_inplace_repeat_lock_held(PyObject *op, Py_ssize_t n) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *self = arrayobject_CAST(op); const Py_ssize_t array_size = Py_SIZE(self); @@ -1155,16 +1245,19 @@ array_inplace_repeat(PyObject *op, Py_ssize_t n) return Py_NewRef(self); } - static PyObject * -ins(arrayobject *self, Py_ssize_t where, PyObject *v) +array_inplace_repeat(PyObject *op, Py_ssize_t n) { - if (ins1(self, where, v) != 0) - return NULL; - Py_RETURN_NONE; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_inplace_repeat_lock_held(op, n); + Py_END_CRITICAL_SECTION(); + return ret; } + /*[clinic input] +@critical_section array.array.count v: object @@ -1174,8 +1267,8 @@ Return number of occurrences of v in the array. [clinic start generated code]*/ static PyObject * -array_array_count(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=3dd3624bf7135a3a input=d9bce9d65e39d1f5]*/ +array_array_count_impl(arrayobject *self, PyObject *v) +/*[clinic end generated code: output=93ead26a2affb739 input=c12c0042c1d0e27e]*/ { Py_ssize_t count = 0; Py_ssize_t i; @@ -1199,6 +1292,7 @@ array_array_count(arrayobject *self, PyObject *v) /*[clinic input] +@critical_section array.array.index v: object @@ -1214,7 +1308,7 @@ Raise ValueError if the value is not present. static PyObject * array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, Py_ssize_t stop) -/*[clinic end generated code: output=c45e777880c99f52 input=089dff7baa7e5a7e]*/ +/*[clinic end generated code: output=c45e777880c99f52 input=fa32ac8ec22175d6]*/ { if (start < 0) { start += Py_SIZE(self); @@ -1247,22 +1341,34 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, } static int -array_contains(PyObject *self, PyObject *v) +array_contains_lock_held(PyObject *op, PyObject *v) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); Py_ssize_t i; int cmp; - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(self); i++) { - PyObject *selfi = getarrayitem(self, i); - if (selfi == NULL) + for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { + PyObject *opi = getarrayitem(op, i); + if (opi == NULL) return -1; - cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); - Py_DECREF(selfi); + cmp = PyObject_RichCompareBool(opi, v, Py_EQ); + Py_DECREF(opi); } return cmp; } +static int +array_contains(PyObject *op, PyObject *v) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_contains_lock_held(op, v); + Py_END_CRITICAL_SECTION(); + return ret; +} + /*[clinic input] +@critical_section array.array.remove v: object @@ -1272,8 +1378,8 @@ Remove the first occurrence of v in the array. [clinic start generated code]*/ static PyObject * -array_array_remove(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=bef06be9fdf9dceb input=0b1e5aed25590027]*/ +array_array_remove_impl(arrayobject *self, PyObject *v) +/*[clinic end generated code: output=f2a24e288ecb2a35 input=78bef3fd40e62f7a]*/ { Py_ssize_t i; @@ -1299,6 +1405,7 @@ array_array_remove(arrayobject *self, PyObject *v) } /*[clinic input] +@critical_section array.array.pop i: Py_ssize_t = -1 @@ -1311,7 +1418,7 @@ i defaults to -1. static PyObject * array_array_pop_impl(arrayobject *self, Py_ssize_t i) -/*[clinic end generated code: output=bc1f0c54fe5308e4 input=8e5feb4c1a11cd44]*/ +/*[clinic end generated code: output=bc1f0c54fe5308e4 input=c69a7f1f8c570e2f]*/ { PyObject *v; @@ -1358,6 +1465,7 @@ array_array_extend_impl(arrayobject *self, PyTypeObject *cls, PyObject *bb) } /*[clinic input] +@critical_section array.array.insert i: Py_ssize_t @@ -1369,12 +1477,15 @@ Insert a new item v into the array before position i. static PyObject * array_array_insert_impl(arrayobject *self, Py_ssize_t i, PyObject *v) -/*[clinic end generated code: output=5a3648e278348564 input=5577d1b4383e9313]*/ +/*[clinic end generated code: output=5a3648e278348564 input=3c922bbd81462978]*/ { - return ins(self, i, v); + if (ins1(self, i, v) != 0) + return NULL; + Py_RETURN_NONE; } /*[clinic input] +@critical_section array.array.buffer_info Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents. @@ -1385,7 +1496,7 @@ the buffer length in bytes. static PyObject * array_array_buffer_info_impl(arrayobject *self) -/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=a58bae5c6e1ac6a6]*/ +/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=9d0dc1ff0e6542e8]*/ { PyObject *retval = NULL, *v; @@ -1411,6 +1522,7 @@ array_array_buffer_info_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.append v: object @@ -1420,13 +1532,16 @@ Append new value v to the end of the array. [clinic start generated code]*/ static PyObject * -array_array_append(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=745a0669bf8db0e2 input=0b98d9d78e78f0fa]*/ +array_array_append_impl(arrayobject *self, PyObject *v) +/*[clinic end generated code: output=2f1e8cbad70c2a8b input=9cdd897c66a40c3f]*/ { - return ins(self, Py_SIZE(self), v); + if (ins1(self, Py_SIZE(self), v) != 0) + return NULL; + Py_RETURN_NONE; } /*[clinic input] +@critical_section array.array.byteswap Byteswap all items of the array. @@ -1437,7 +1552,7 @@ raised. static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=6a85591b950a0186]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=e691b6eff94d8b2e]*/ { char *p; Py_ssize_t i; @@ -1487,6 +1602,7 @@ array_array_byteswap_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.reverse Reverse the order of the items in the array. @@ -1494,7 +1610,7 @@ Reverse the order of the items in the array. static PyObject * array_array_reverse_impl(arrayobject *self) -/*[clinic end generated code: output=c04868b36f6f4089 input=cd904f01b27d966a]*/ +/*[clinic end generated code: output=c04868b36f6f4089 input=e3947e98aed068ed]*/ { Py_ssize_t itemsize = self->ob_descr->itemsize; char *p, *q; @@ -1584,6 +1700,7 @@ array_array_fromfile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f, } /*[clinic input] +@critical_section array.array.tofile cls: defining_class @@ -1595,7 +1712,7 @@ Write all items (as machine values) to the file object f. static PyObject * array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) -/*[clinic end generated code: output=4560c628d9c18bc2 input=5a24da7a7b407b52]*/ +/*[clinic end generated code: output=4560c628d9c18bc2 input=a26bc66df57864dd]*/ { Py_ssize_t nbytes = Py_SIZE(self) * self->ob_descr->itemsize; /* Write 64K blocks at a time */ @@ -1628,11 +1745,12 @@ array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) Py_DECREF(res); /* drop write result */ } - done: +done: Py_RETURN_NONE; } /*[clinic input] +@critical_section self list array.array.fromlist list: object @@ -1642,8 +1760,8 @@ Append items to array from list. [clinic start generated code]*/ static PyObject * -array_array_fromlist(arrayobject *self, PyObject *list) -/*[clinic end generated code: output=26411c2d228a3e3f input=be2605a96c49680f]*/ +array_array_fromlist_impl(arrayobject *self, PyObject *list) +/*[clinic end generated code: output=6c23733a68dd68df input=c7c056aaf85d997a]*/ { Py_ssize_t n; @@ -1676,6 +1794,7 @@ array_array_fromlist(arrayobject *self, PyObject *list) } /*[clinic input] +@critical_section array.array.tolist Convert array to an ordinary list with the same items. @@ -1683,7 +1802,7 @@ Convert array to an ordinary list with the same items. static PyObject * array_array_tolist_impl(arrayobject *self) -/*[clinic end generated code: output=00b60cc9eab8ef89 input=a8d7784a94f86b53]*/ +/*[clinic end generated code: output=00b60cc9eab8ef89 input=4543fdbac475c52c]*/ { PyObject *list = PyList_New(Py_SIZE(self)); Py_ssize_t i; @@ -1703,19 +1822,29 @@ array_array_tolist_impl(arrayobject *self) return NULL; } + +/*[clinic input] +@critical_section +array.array.frombytes + + buffer: Py_buffer + / + +Appends items from the string, interpreting it as an array of machine values, as if it had been read from a file using the fromfile() method. +[clinic start generated code]*/ + static PyObject * -frombytes(arrayobject *self, Py_buffer *buffer) +array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) +/*[clinic end generated code: output=d9842c8f7510a516 input=2245f9ea58579960]*/ { int itemsize = self->ob_descr->itemsize; Py_ssize_t n; if (buffer->itemsize != 1) { - PyBuffer_Release(buffer); PyErr_SetString(PyExc_TypeError, "a bytes-like object is required"); return NULL; } n = buffer->len; if (n % itemsize != 0) { - PyBuffer_Release(buffer); PyErr_SetString(PyExc_ValueError, "bytes length not a multiple of item size"); return NULL; @@ -1725,37 +1854,19 @@ frombytes(arrayobject *self, Py_buffer *buffer) Py_ssize_t old_size = Py_SIZE(self); if ((n > PY_SSIZE_T_MAX - old_size) || ((old_size + n) > PY_SSIZE_T_MAX / itemsize)) { - PyBuffer_Release(buffer); return PyErr_NoMemory(); } if (array_resize(self, old_size + n) == -1) { - PyBuffer_Release(buffer); return NULL; } memcpy(self->ob_item + old_size * itemsize, buffer->buf, n * itemsize); } - PyBuffer_Release(buffer); Py_RETURN_NONE; } /*[clinic input] -array.array.frombytes - - buffer: Py_buffer - / - -Appends items from the string, interpreting it as an array of machine values, as if it had been read from a file using the fromfile() method. -[clinic start generated code]*/ - -static PyObject * -array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) -/*[clinic end generated code: output=d9842c8f7510a516 input=378db226dfac949e]*/ -{ - return frombytes(self, buffer); -} - -/*[clinic input] +@critical_section array.array.tobytes Convert the array to an array of machine values and return the bytes representation. @@ -1763,7 +1874,7 @@ Convert the array to an array of machine values and return the bytes representat static PyObject * array_array_tobytes_impl(arrayobject *self) -/*[clinic end generated code: output=87318e4edcdc2bb6 input=90ee495f96de34f5]*/ +/*[clinic end generated code: output=87318e4edcdc2bb6 input=c4d44d5499d2320f]*/ { if (Py_SIZE(self) <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { return PyBytes_FromStringAndSize(self->ob_item, @@ -1774,6 +1885,7 @@ array_array_tobytes_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.fromunicode ustr: unicode @@ -1788,7 +1900,7 @@ some other type. static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) -/*[clinic end generated code: output=24359f5e001a7f2b input=025db1fdade7a4ce]*/ +/*[clinic end generated code: output=24359f5e001a7f2b input=01e2a776cee82011]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -1836,6 +1948,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) } /*[clinic input] +@critical_section array.array.tounicode Extends this array with data from the unicode string ustr. @@ -1847,7 +1960,7 @@ unicode string from an array of some other type. static PyObject * array_array_tounicode_impl(arrayobject *self) -/*[clinic end generated code: output=08e442378336e1ef input=127242eebe70b66d]*/ +/*[clinic end generated code: output=08e442378336e1ef input=6c69dfe81a279b91]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -1876,7 +1989,8 @@ array_array___sizeof___impl(arrayobject *self) /*[clinic end generated code: output=d8e1c61ebbe3eaed input=805586565bf2b3c6]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)self->allocated * (size_t)self->ob_descr->itemsize; + res += (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated) + * (size_t)self->ob_descr->itemsize; return PyLong_FromSize_t(res); } @@ -2271,6 +2385,7 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, } /*[clinic input] +@critical_section array.array.__reduce_ex__ cls: defining_class @@ -2283,7 +2398,7 @@ Return state information for pickling. static PyObject * array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, PyObject *value) -/*[clinic end generated code: output=4958ee5d79452ad5 input=19968cf0f91d3eea]*/ +/*[clinic end generated code: output=4958ee5d79452ad5 input=18c90a4cad7ac527]*/ { PyObject *dict; PyObject *result; @@ -2411,8 +2526,9 @@ static PyMethodDef array_methods[] = { }; static PyObject * -array_repr(PyObject *op) +array_repr_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); char typecode; PyObject *s, *v = NULL; Py_ssize_t len; @@ -2438,9 +2554,20 @@ array_repr(PyObject *op) return s; } +static PyObject * +array_repr(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_repr_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + static PyObject* -array_subscr(PyObject *op, PyObject *item) +array_subscr_lock_held(PyObject *op, PyObject *item) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *self = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(self)); @@ -2502,12 +2629,28 @@ array_subscr(PyObject *op, PyObject *item) } } +static PyObject * +array_subscr(PyObject *op, PyObject *item) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_subscr_lock_held(op, item); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int -array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) +array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) { - Py_ssize_t start, stop, step, slicelength, needed; + array_state* state = find_array_state_by_type(Py_TYPE(op)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); +#ifdef Py_DEBUG + if (value != NULL && array_Check(value, state)) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(value); + } +#endif arrayobject *self = arrayobject_CAST(op); - array_state* state = find_array_state_by_type(Py_TYPE(self)); + Py_ssize_t start, stop, step, slicelength, needed; arrayobject* other; int itemsize; @@ -2558,7 +2701,7 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) value = array_slice(other, 0, needed); if (value == NULL) return -1; - ret = array_ass_subscr(op, item, value); + ret = array_ass_subscr_lock_held(op, item, value); Py_DECREF(value); return ret; } @@ -2661,19 +2804,38 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) } } +static int +array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) +{ + int ret; + array_state* state = find_array_state_by_type(Py_TYPE(op)); + if (value != NULL && array_Check(value, state)) { + Py_BEGIN_CRITICAL_SECTION2(op, value); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION(); + } + return ret; +} + static const void *emptybuf = ""; static int -array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +array_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + arrayobject *self = arrayobject_CAST(op); if (view == NULL) { PyErr_SetString(PyExc_BufferError, "array_buffer_getbuf: view==NULL argument is obsolete"); return -1; } - arrayobject *self = arrayobject_CAST(op); view->buf = (void *)self->ob_item; view->obj = Py_NewRef(self); if (view->buf == NULL) @@ -2705,61 +2867,39 @@ array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) return 0; } +static int +array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_buffer_getbuf_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void array_buffer_relbuf(PyObject *op, Py_buffer *Py_UNUSED(view)) { + Py_BEGIN_CRITICAL_SECTION(op); arrayobject *self = arrayobject_CAST(op); self->ob_exports--; + assert(self->ob_exports >= 0); + Py_END_CRITICAL_SECTION(); } static PyObject * -array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) { +#ifdef Py_DEBUG + if (initial != NULL) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(initial); + } +#endif array_state *state = find_array_state_by_type(type); - int c; - PyObject *initial = NULL, *it = NULL; + PyObject *it = NULL; const struct arraydescr *descr; - - if ((type == state->ArrayType || - type->tp_init == state->ArrayType->tp_init) && - !_PyArg_NoKeywords("array.array", kwds)) - return NULL; - - if (!PyArg_ParseTuple(args, "C|O:array", &c, &initial)) - return NULL; - - if (PySys_Audit("array.__new__", "CO", - c, initial ? initial : Py_None) < 0) { - return NULL; - } - - if (c == 'u') { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "The 'u' type code is deprecated and " - "will be removed in Python 3.16", - 1)) { - return NULL; - } - } - bool is_unicode = c == 'u' || c == 'w'; - if (initial && !is_unicode) { - if (PyUnicode_Check(initial)) { - PyErr_Format(PyExc_TypeError, "cannot use a str to initialize " - "an array with typecode '%c'", c); - return NULL; - } - else if (array_Check(initial, state)) { - int ic = ((arrayobject*)initial)->ob_descr->typecode; - if (ic == 'u' || ic == 'w') { - PyErr_Format(PyExc_TypeError, "cannot use a unicode array to " - "initialize an array with typecode '%c'", c); - return NULL; - } - } - } - if (!(initial == NULL || PyList_Check(initial) || PyByteArray_Check(initial) || PyBytes_Check(initial) @@ -2877,6 +3017,69 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } +static PyObject * +array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + array_state *state = find_array_state_by_type(type); + int c; + PyObject *initial = NULL; + + if ((type == state->ArrayType || + type->tp_init == state->ArrayType->tp_init) && + !_PyArg_NoKeywords("array.array", kwds)) { + return NULL; + } + + if (!PyArg_ParseTuple(args, "C|O:array", &c, &initial)) { + return NULL; + } + + if (PySys_Audit("array.__new__", "CO", + c, initial ? initial : Py_None) < 0) { + return NULL; + } + + if (c == 'u') { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "The 'u' type code is deprecated and " + "will be removed in Python 3.16", + 1)) { + return NULL; + } + } + + bool is_unicode = c == 'u' || c == 'w'; + + if (initial && !is_unicode) { + if (PyUnicode_Check(initial)) { + PyErr_Format(PyExc_TypeError, "cannot use a str to initialize " + "an array with typecode '%c'", c); + return NULL; + } + else if (array_Check(initial, state)) { + int ic = ((arrayobject*)initial)->ob_descr->typecode; + if (ic == 'u' || ic == 'w') { + PyErr_Format(PyExc_TypeError, "cannot use a unicode array to " + "initialize an array with typecode '%c'", c); + return NULL; + } + } + } + + PyObject *ret; + + if (initial == NULL) { + ret = array_new_internal_lock_held(type, initial, c); + } + else { + Py_BEGIN_CRITICAL_SECTION(initial); + ret = array_new_internal_lock_held(type, initial, c); + Py_END_CRITICAL_SECTION(); + } + + return ret; +} + PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ @@ -3019,7 +3222,7 @@ array_iter(PyObject *op) return NULL; it->ao = (arrayobject*)Py_NewRef(ao); - it->index = 0; + it->index = 0; // -1 indicates exhausted it->getitem = ao->ob_descr->getitem; PyObject_GC_Track(it); return (PyObject *)it; @@ -3030,23 +3233,37 @@ arrayiter_next(PyObject *op) { arrayiterobject *it = arrayiterobject_CAST(op); assert(it != NULL); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->index); + if (index < 0) { + return NULL; + } + PyObject *ret; + arrayobject *ao = it->ao; #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(it)); assert(PyObject_TypeCheck(it, state->ArrayIterType)); + assert(array_Check(ao, state)); #endif - arrayobject *ao = it->ao; - if (ao == NULL) { - return NULL; + + Py_BEGIN_CRITICAL_SECTION(ao); + if (index < Py_SIZE(ao)) { + ret = (*it->getitem)(ao, index); } -#ifndef NDEBUG - assert(array_Check(ao, state)); + else { + ret = NULL; + } + Py_END_CRITICAL_SECTION(); + + if (ret != NULL) { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, index + 1); + } + else { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, -1); +#ifndef Py_GIL_DISABLED + Py_CLEAR(it->ao); #endif - if (it->index < Py_SIZE(ao)) { - return (*it->getitem)(ao, it->index++); } - it->ao = NULL; - Py_DECREF(ao); - return NULL; + return ret; } static void @@ -3082,14 +3299,14 @@ static PyObject * array_arrayiterator___reduce___impl(arrayiterobject *self, PyTypeObject *cls) /*[clinic end generated code: output=4b032417a2c8f5e6 input=ac64e65a87ad452e]*/ { - array_state *state = get_array_state_by_class(cls); assert(state != NULL); PyObject *func = _PyEval_GetBuiltin(state->str_iter); - if (self->ao == NULL) { - return Py_BuildValue("N(())", func); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->index); + if (index >= 0) { + return Py_BuildValue("N(O)n", func, self->ao, index); } - return Py_BuildValue("N(O)n", func, self->ao, self->index); + return Py_BuildValue("N(())", func); } /*[clinic input] @@ -3106,17 +3323,20 @@ array_arrayiterator___setstate__(arrayiterobject *self, PyObject *state) /*[clinic end generated code: output=397da9904e443cbe input=f47d5ceda19e787b]*/ { Py_ssize_t index = PyLong_AsSsize_t(state); - if (index == -1 && PyErr_Occurred()) + if (index == -1 && PyErr_Occurred()) { return NULL; - arrayobject *ao = self->ao; - if (ao != NULL) { - if (index < 0) { - index = 0; + } + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->index) >= 0) { + if (index < -1) { + index = -1; } - else if (index > Py_SIZE(ao)) { - index = Py_SIZE(ao); /* iterator exhausted */ + else { + Py_ssize_t size = Pyarrayobject_GET_SIZE(self->ao); + if (index > size) { + index = size; /* iterator at end */ + } } - self->index = index; + FT_ATOMIC_STORE_SSIZE_RELAXED(self->index, index); } Py_RETURN_NONE; } diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index c5b62b16699d06..3816bb7709658e 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(array_array_clear__doc__, @@ -23,7 +24,13 @@ array_array_clear_impl(arrayobject *self); static PyObject * array_array_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_clear_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_clear_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___copy____doc__, @@ -41,7 +48,13 @@ array_array___copy___impl(arrayobject *self); static PyObject * array_array___copy__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array___copy___impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array___copy___impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___deepcopy____doc__, @@ -53,6 +66,21 @@ PyDoc_STRVAR(array_array___deepcopy____doc__, #define ARRAY_ARRAY___DEEPCOPY___METHODDEF \ {"__deepcopy__", (PyCFunction)array_array___deepcopy__, METH_O, array_array___deepcopy____doc__}, +static PyObject * +array_array___deepcopy___impl(arrayobject *self, PyObject *unused); + +static PyObject * +array_array___deepcopy__(arrayobject *self, PyObject *unused) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array___deepcopy___impl((arrayobject *)self, unused); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(array_array_count__doc__, "count($self, v, /)\n" "--\n" @@ -62,6 +90,21 @@ PyDoc_STRVAR(array_array_count__doc__, #define ARRAY_ARRAY_COUNT_METHODDEF \ {"count", (PyCFunction)array_array_count, METH_O, array_array_count__doc__}, +static PyObject * +array_array_count_impl(arrayobject *self, PyObject *v); + +static PyObject * +array_array_count(arrayobject *self, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_count_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(array_array_index__doc__, "index($self, v, start=0, stop=sys.maxsize, /)\n" "--\n" @@ -102,7 +145,9 @@ array_array_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_index_impl((arrayobject *)self, v, start, stop); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -117,6 +162,21 @@ PyDoc_STRVAR(array_array_remove__doc__, #define ARRAY_ARRAY_REMOVE_METHODDEF \ {"remove", (PyCFunction)array_array_remove, METH_O, array_array_remove__doc__}, +static PyObject * +array_array_remove_impl(arrayobject *self, PyObject *v); + +static PyObject * +array_array_remove(arrayobject *self, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_remove_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(array_array_pop__doc__, "pop($self, i=-1, /)\n" "--\n" @@ -156,7 +216,9 @@ array_array_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs) i = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_pop_impl((arrayobject *)self, i); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -241,7 +303,9 @@ array_array_insert(PyObject *self, PyObject *const *args, Py_ssize_t nargs) i = ival; } v = args[1]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_insert_impl((arrayobject *)self, i, v); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -265,7 +329,13 @@ array_array_buffer_info_impl(arrayobject *self); static PyObject * array_array_buffer_info(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_buffer_info_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_buffer_info_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_append__doc__, @@ -277,6 +347,21 @@ PyDoc_STRVAR(array_array_append__doc__, #define ARRAY_ARRAY_APPEND_METHODDEF \ {"append", (PyCFunction)array_array_append, METH_O, array_array_append__doc__}, +static PyObject * +array_array_append_impl(arrayobject *self, PyObject *v); + +static PyObject * +array_array_append(arrayobject *self, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_append_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(array_array_byteswap__doc__, "byteswap($self, /)\n" "--\n" @@ -295,7 +380,13 @@ array_array_byteswap_impl(arrayobject *self); static PyObject * array_array_byteswap(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_byteswap_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_byteswap_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_reverse__doc__, @@ -313,7 +404,13 @@ array_array_reverse_impl(arrayobject *self); static PyObject * array_array_reverse(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_reverse_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_reverse_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_fromfile__doc__, @@ -412,7 +509,9 @@ array_array_tofile(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ goto exit; } f = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_tofile_impl((arrayobject *)self, cls, f); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -427,6 +526,21 @@ PyDoc_STRVAR(array_array_fromlist__doc__, #define ARRAY_ARRAY_FROMLIST_METHODDEF \ {"fromlist", (PyCFunction)array_array_fromlist, METH_O, array_array_fromlist__doc__}, +static PyObject * +array_array_fromlist_impl(arrayobject *self, PyObject *list); + +static PyObject * +array_array_fromlist(arrayobject *self, PyObject *list) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(self, list); + return_value = array_array_fromlist_impl((arrayobject *)self, list); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(array_array_tolist__doc__, "tolist($self, /)\n" "--\n" @@ -442,7 +556,13 @@ array_array_tolist_impl(arrayobject *self); static PyObject * array_array_tolist(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tolist_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tolist_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_frombytes__doc__, @@ -466,7 +586,9 @@ array_array_frombytes(PyObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_frombytes_impl((arrayobject *)self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -492,7 +614,13 @@ array_array_tobytes_impl(arrayobject *self); static PyObject * array_array_tobytes(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tobytes_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tobytes_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_fromunicode__doc__, @@ -522,7 +650,9 @@ array_array_fromunicode(PyObject *self, PyObject *arg) goto exit; } ustr = arg; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_fromunicode_impl((arrayobject *)self, ustr); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -547,7 +677,13 @@ array_array_tounicode_impl(arrayobject *self); static PyObject * array_array_tounicode(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tounicode_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tounicode_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___sizeof____doc__, @@ -659,7 +795,9 @@ array_array___reduce_ex__(PyObject *self, PyTypeObject *cls, PyObject *const *ar goto exit; } value = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array___reduce_ex___impl((arrayobject *)self, cls, value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -695,4 +833,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=8120dc5c4fa414b9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c9219e074c62e0c8 input=a9049054013a1b77]*/ From 4c71f00624330c850d71461521f3ef9fc17eab8c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 2 Mar 2025 15:56:54 -0500 Subject: [PATCH 02/32] lockfree read and write single element --- Include/internal/pycore_pymem.h | 2 +- Modules/arraymodule.c | 538 +++++++++++++++++++++++--------- 2 files changed, 388 insertions(+), 152 deletions(-) diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 5386d4c5f83031..bcd13d80cba975 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -117,7 +117,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str); extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. -extern void _PyMem_FreeDelayed(void *ptr); +PyAPI_FUNC(void) _PyMem_FreeDelayed(void *ptr); // Enqueue an object to be freed possibly after some delay #ifdef Py_GIL_DISABLED diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 0775b26e1d68ed..41a5371acaf125 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -14,6 +14,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_pymem.h" // _PyMem_FreeDelayed #include // offsetof() #include @@ -33,8 +34,8 @@ static struct PyModuleDef arraymodule; struct arraydescr { char typecode; int itemsize; - PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); - int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); + PyObject * (*getitem)(char *, Py_ssize_t); + int (*setitem)(char *, Py_ssize_t, PyObject *); int (*compareitems)(const void *, const void *, Py_ssize_t); const char *formats; int is_integer_type; @@ -54,7 +55,6 @@ typedef struct { PyObject_HEAD Py_ssize_t index; arrayobject *ao; - PyObject* (*getitem)(struct arrayobject *, Py_ssize_t); } arrayiterobject; typedef struct { @@ -144,6 +144,85 @@ enum machine_format_code { #define array_Check(op, state) PyObject_TypeCheck(op, state->ArrayType) +#ifdef Py_GIL_DISABLED +typedef struct { + Py_ssize_t allocated; + char ob_item[]; +} _PyArrayArray; +#endif + +static char * +arrayarray_alloc(Py_ssize_t size, int itemsize) +{ +#ifdef Py_GIL_DISABLED + if (size > PY_SSIZE_T_MAX/itemsize - 1) { + return NULL; + } + _PyArrayArray *array = PyMem_Malloc(sizeof(_PyArrayArray) + size * itemsize); + if (array == NULL) { + return NULL; + } + array->allocated = size; + return array->ob_item; +#else + return PyMem_Malloc(size * itemsize); +#endif +} + +static void +arrayarray_free(char *items, bool use_qsbr) +{ +#ifdef Py_GIL_DISABLED + if (items == NULL) { + return; + } + _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); + if (use_qsbr) { + _PyMem_FreeDelayed(array); + } + else { + PyMem_Free(array); + } +#else + PyMem_Free(items); +#endif +} + +#ifdef Py_GIL_DISABLED + +static Py_ssize_t +arrayarray_capacity(char *items) +{ + _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); + return array->allocated; +} + +static char * +arrayarray_set(char *items, char *newitems, Py_ssize_t newsize, int itemsize, bool use_qsbr) +{ + arrayarray_free(items, use_qsbr); + items = arrayarray_alloc(newsize, itemsize); + if (items != NULL) { + memcpy(items, newitems, newsize * itemsize); + } + return items; +} + +static void +ensure_shared_on_resize(arrayobject *self) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + // Ensure that the array is freed using QSBR if we are not the + // owning thread. + if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && + !_PyObject_GC_IS_SHARED(self)) + { + _PyObject_GC_SET_SHARED(self); + } +} + +# endif // Py_GIL_DISABLED + static int array_resize(arrayobject *self, Py_ssize_t newsize) { @@ -170,7 +249,11 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } if (newsize == 0) { - PyMem_Free(self->ob_item); +#ifdef Py_GIL_DISABLED + arrayarray_free(self->ob_item, _PyObject_GC_IS_SHARED(self)); +#else + arrayarray_free(self->ob_item, false); +#endif self->ob_item = NULL; Py_SET_SIZE(self, 0); FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, 0); @@ -191,17 +274,37 @@ array_resize(arrayobject *self, Py_ssize_t newsize) _new_size = (newsize >> 4) + (Py_SIZE(self) < 8 ? 3 : 7) + newsize; items = self->ob_item; + int itemsize = self->ob_descr->itemsize; + /* XXX The following multiplication and division does not optimize away like it does for lists since the size is not known at compile time */ - if (_new_size <= ((~(size_t)0) / self->ob_descr->itemsize)) - PyMem_RESIZE(items, char, (_new_size * self->ob_descr->itemsize)); - else - items = NULL; + if (_new_size > ((~(size_t)0) / itemsize)) { + PyErr_NoMemory(); + return -1; + } + +#ifdef Py_GIL_DISABLED + ensure_shared_on_resize(self); + char *newitems = arrayarray_alloc(_new_size, itemsize); + if (newitems == NULL) { + PyErr_NoMemory(); + return -1; + } + if (items != NULL) { + Py_ssize_t size = Py_SIZE(self); + memcpy(newitems, items, Py_MIN(size, newsize) * itemsize); + arrayarray_free(items, _PyObject_GC_IS_SHARED(self)); + } + _Py_atomic_store_ptr_release(&self->ob_item, newitems); +#else + PyMem_RESIZE(items, char, (_new_size * itemsize)); if (items == NULL) { PyErr_NoMemory(); return -1; } self->ob_item = items; +#endif + Py_SET_SIZE(self, newsize); FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, _new_size); return 0; @@ -220,14 +323,14 @@ in bounds; that's the responsibility of the caller. ****************************************************************************/ static PyObject * -b_getitem(arrayobject *ap, Py_ssize_t i) +b_getitem(char *ob_item, Py_ssize_t i) { - long x = ((signed char *)ap->ob_item)[i]; + long x = ((signed char *)ob_item)[i]; return PyLong_FromLong(x); } static int -b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +b_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { short x; /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore @@ -246,37 +349,37 @@ b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((char *)ap->ob_item)[i] = (char)x; + ((char *)ob_item)[i] = (char)x; return 0; } static PyObject * -BB_getitem(arrayobject *ap, Py_ssize_t i) +BB_getitem(char *ob_item, Py_ssize_t i) { - long x = ((unsigned char *)ap->ob_item)[i]; + long x = ((unsigned char *)ob_item)[i]; return PyLong_FromLong(x); } static int -BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +BB_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; if (i >= 0) - ((unsigned char *)ap->ob_item)[i] = x; + ((unsigned char *)ob_item)[i] = x; return 0; } static PyObject * -u_getitem(arrayobject *ap, Py_ssize_t i) +u_getitem(char *ob_item, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((wchar_t *) ap->ob_item)[i]); + return PyUnicode_FromOrdinal(((wchar_t *) ob_item)[i]); } static int -u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +u_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -307,19 +410,19 @@ u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { - ((wchar_t *)ap->ob_item)[i] = w; + ((wchar_t *)ob_item)[i] = w; } return 0; } static PyObject * -w_getitem(arrayobject *ap, Py_ssize_t i) +w_getitem(char *ob_item, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((Py_UCS4 *) ap->ob_item)[i]); + return PyUnicode_FromOrdinal(((Py_UCS4 *) ob_item)[i]); } static int -w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +w_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -337,38 +440,38 @@ w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } if (i >= 0) { - ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(v, 0); + ((Py_UCS4 *)ob_item)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } static PyObject * -h_getitem(arrayobject *ap, Py_ssize_t i) +h_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLong((long) ((short *)ap->ob_item)[i]); + return PyLong_FromLong((long) ((short *)ob_item)[i]); } static int -h_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +h_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; if (i >= 0) - ((short *)ap->ob_item)[i] = x; + ((short *)ob_item)[i] = x; return 0; } static PyObject * -HH_getitem(arrayobject *ap, Py_ssize_t i) +HH_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLong((long) ((unsigned short *)ap->ob_item)[i]); + return PyLong_FromLong((long) ((unsigned short *)ob_item)[i]); } static int -HH_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +HH_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore @@ -386,37 +489,37 @@ HH_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((short *)ap->ob_item)[i] = (short)x; + ((short *)ob_item)[i] = (short)x; return 0; } static PyObject * -i_getitem(arrayobject *ap, Py_ssize_t i) +i_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLong((long) ((int *)ap->ob_item)[i]); + return PyLong_FromLong((long) ((int *)ob_item)[i]); } static int -i_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +i_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; if (i >= 0) - ((int *)ap->ob_item)[i] = x; + ((int *)ob_item)[i] = x; return 0; } static PyObject * -II_getitem(arrayobject *ap, Py_ssize_t i) +II_getitem(char *ob_item, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) ((unsigned int *)ap->ob_item)[i]); + (unsigned long) ((unsigned int *)ob_item)[i]); } static int -II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +II_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -444,7 +547,7 @@ II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + ((unsigned int *)ob_item)[i] = (unsigned int)x; if (do_decref) { Py_DECREF(v); @@ -453,30 +556,30 @@ II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } static PyObject * -l_getitem(arrayobject *ap, Py_ssize_t i) +l_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLong(((long *)ap->ob_item)[i]); + return PyLong_FromLong(((long *)ob_item)[i]); } static int -l_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +l_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { long x; if (!PyArg_Parse(v, "l;array item must be integer", &x)) return -1; if (i >= 0) - ((long *)ap->ob_item)[i] = x; + ((long *)ob_item)[i] = x; return 0; } static PyObject * -LL_getitem(arrayobject *ap, Py_ssize_t i) +LL_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromUnsignedLong(((unsigned long *)ap->ob_item)[i]); + return PyLong_FromUnsignedLong(((unsigned long *)ob_item)[i]); } static int -LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +LL_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -496,7 +599,7 @@ LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned long *)ap->ob_item)[i] = x; + ((unsigned long *)ob_item)[i] = x; if (do_decref) { Py_DECREF(v); @@ -505,31 +608,31 @@ LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } static PyObject * -q_getitem(arrayobject *ap, Py_ssize_t i) +q_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLongLong(((long long *)ap->ob_item)[i]); + return PyLong_FromLongLong(((long long *)ob_item)[i]); } static int -q_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +q_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { long long x; if (!PyArg_Parse(v, "L;array item must be integer", &x)) return -1; if (i >= 0) - ((long long *)ap->ob_item)[i] = x; + ((long long *)ob_item)[i] = x; return 0; } static PyObject * -QQ_getitem(arrayobject *ap, Py_ssize_t i) +QQ_getitem(char *ob_item, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - ((unsigned long long *)ap->ob_item)[i]); + ((unsigned long long *)ob_item)[i]); } static int -QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +QQ_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { unsigned long long x; int do_decref = 0; /* if nb_int was called */ @@ -549,7 +652,7 @@ QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned long long *)ap->ob_item)[i] = x; + ((unsigned long long *)ob_item)[i] = x; if (do_decref) { Py_DECREF(v); @@ -558,36 +661,36 @@ QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } static PyObject * -f_getitem(arrayobject *ap, Py_ssize_t i) +f_getitem(char *ob_item, Py_ssize_t i) { - return PyFloat_FromDouble((double) ((float *)ap->ob_item)[i]); + return PyFloat_FromDouble((double) ((float *)ob_item)[i]); } static int -f_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +f_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { float x; if (!PyArg_Parse(v, "f;array item must be float", &x)) return -1; if (i >= 0) - ((float *)ap->ob_item)[i] = x; + ((float *)ob_item)[i] = x; return 0; } static PyObject * -d_getitem(arrayobject *ap, Py_ssize_t i) +d_getitem(char *ob_item, Py_ssize_t i) { - return PyFloat_FromDouble(((double *)ap->ob_item)[i]); + return PyFloat_FromDouble(((double *)ob_item)[i]); } static int -d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +d_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { double x; if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; if (i >= 0) - ((double *)ap->ob_item)[i] = x; + ((double *)ob_item)[i] = x; return 0; } @@ -650,7 +753,6 @@ static PyObject * newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *descr) { arrayobject *op; - size_t nbytes; if (size < 0) { PyErr_BadInternalCall(); @@ -661,7 +763,6 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des if (size > PY_SSIZE_T_MAX / descr->itemsize) { return PyErr_NoMemory(); } - nbytes = size * descr->itemsize; op = (arrayobject *) type->tp_alloc(type, 0); if (op == NULL) { return NULL; @@ -674,7 +775,7 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des op->ob_item = NULL; } else { - op->ob_item = PyMem_NEW(char, nbytes); + op->ob_item = arrayarray_alloc(size, descr->itemsize); if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); @@ -684,20 +785,149 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return (PyObject *) op; } +static inline int +valid_index(Py_ssize_t i, Py_ssize_t limit) +{ + return (size_t) i < (size_t) limit; +} + static PyObject * -getarrayitem(PyObject *op, Py_ssize_t i) +getarrayitem(PyObject *op, Py_ssize_t i, char *items) +{ +#ifndef NDEBUG + array_state *state = find_array_state_by_type(Py_TYPE(op)); + assert(array_Check(op, state)); +#endif + arrayobject *ap = arrayobject_CAST(op); + assert(valid_index(i, Py_SIZE(op))); + return (*ap->ob_descr->getitem)(items, i); +} + +static PyObject * +getarrayitem_locked(PyObject *op, Py_ssize_t i) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); +#ifdef Py_GIL_DISABLED + if (!_PyObject_GC_IS_SHARED(op)) { + _PyObject_GC_SET_SHARED(op); + } +#endif + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + arrayobject *ap = (arrayobject *)op; + ret = getarrayitem(op, i, ap->ob_item); + Py_END_CRITICAL_SECTION(); + return ret; +} + +#ifdef Py_GIL_DISABLED + +static PyObject * +getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) +{ + if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { + return getarrayitem_locked(op, i); + } + Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + if (!valid_index(i, size)) { + return NULL; + } + arrayobject *ap = (arrayobject *)op; + char *items = _Py_atomic_load_ptr(&ap->ob_item); + if (items == NULL) { + return NULL; + } + Py_ssize_t cap = arrayarray_capacity(items); + if (!valid_index(i, cap)) { + return NULL; + } + return getarrayitem(op, i, items); +} + +#else // Py_GIL_DISABLED + +static PyObject * +getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) +{ + return getarrayitem_locked(op, i); +} + +#endif // Py_GIL_DISABLED + +static int +setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v, char *items) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(op)); assert(array_Check(op, state)); #endif - arrayobject *ap; - ap = (arrayobject *)op; - assert(i>=0 && iob_descr->getitem)(ap, i); + arrayobject *ap = arrayobject_CAST(op); + assert(valid_index(i, Py_SIZE(op))); + return (*ap->ob_descr->setitem)(items, i, v); } +static int +setarrayitem_locked(PyObject *op, Py_ssize_t i, PyObject *v) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(op); +#ifdef Py_GIL_DISABLED + if (!_PyObject_GC_IS_SHARED(op)) { + _PyObject_GC_SET_SHARED(op); + } +#endif + if (!valid_index(i, Py_SIZE(op))) { + PyErr_SetString(PyExc_IndexError, "array index out of range"); + ret = -1; + } + else { + arrayobject *ap = (arrayobject *)op; + ret = setarrayitem(op, i, v, ap->ob_item); + } + Py_END_CRITICAL_SECTION(); + return ret; +} + +#ifdef Py_GIL_DISABLED + +static int +setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) +{ + if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { + return setarrayitem_locked(op, i, v); + } + Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + if (!valid_index(i, size)) { + goto error; + } + arrayobject *ap = (arrayobject *)op; + char *items = _Py_atomic_load_ptr(&ap->ob_item); + if (items == NULL) { + goto error; + } + Py_ssize_t cap = arrayarray_capacity(items); + if (!valid_index(i, cap)) { + goto error; + } + return setarrayitem(op, i, v, items); + +error: + PyErr_SetString(PyExc_IndexError, "array index out of range"); + return -1; +} + +#else // Py_GIL_DISABLED + +static int +setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) +{ + return setarrayitem_locked(op, i, v); +} + +#endif // Py_GIL_DISABLED + static int ins1(arrayobject *self, Py_ssize_t where, PyObject *v) { @@ -708,7 +938,7 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) PyErr_BadInternalCall(); return -1; } - if ((*self->ob_descr->setitem)(self, -1, v) < 0) + if ((*self->ob_descr->setitem)(self->ob_item, -1, v) < 0) return -1; if (array_resize(self, n+1) == -1) @@ -726,7 +956,7 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) memmove(items + (where+1)*self->ob_descr->itemsize, items + where*self->ob_descr->itemsize, (n-where)*self->ob_descr->itemsize); - return (*self->ob_descr->setitem)(self, where, v); + return (*self->ob_descr->setitem)(self->ob_item, where, v); } /* Methods */ @@ -754,7 +984,7 @@ array_dealloc(PyObject *op) PyObject_ClearWeakRefs(op); } if (self->ob_item != NULL) { - PyMem_Free(self->ob_item); + arrayarray_free(self->ob_item, false); } tp->tp_free(op); Py_DECREF(tp); @@ -814,11 +1044,11 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(v, i); + vi = getarrayitem(v, i, va->ob_item); if (vi == NULL) { return NULL; } - wi = getarrayitem(w, i); + wi = getarrayitem(w, i, wa->ob_item); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -889,26 +1119,17 @@ array_length(PyObject *op) return Pyarrayobject_GET_SIZE(self); } + static PyObject * -array_item_lock_held(PyObject *op, Py_ssize_t i) +array_item(PyObject *op, Py_ssize_t i) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - if (i < 0 || i >= Py_SIZE(op)) { + PyObject *item = getarrayitem_maybe_locked(op, i); + if (item == NULL) { PyErr_SetString(PyExc_IndexError, "array index out of range"); - return NULL; } - return getarrayitem(op, i); + return item; } -static PyObject * -array_item(PyObject *op, Py_ssize_t i) -{ - PyObject *ret; - Py_BEGIN_CRITICAL_SECTION(op); - ret = array_item_lock_held(op, i); - Py_END_CRITICAL_SECTION(); - return ret; -} static PyObject * array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) @@ -1107,27 +1328,15 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) return 0; } -static int -setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v) -{ - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - arrayobject *a = arrayobject_CAST(op); - if (i < 0 || i >= Py_SIZE(a)) { - PyErr_SetString(PyExc_IndexError, - "array assignment index out of range"); - return -1; - } - if (v == NULL) - return array_del_slice(a, i, i+1); - return (*a->ob_descr->setitem)(a, i, v); -} - static int array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) { + if (v != NULL) { + return setarrayitem_maybe_locked(op, i, v); + } int ret; Py_BEGIN_CRITICAL_SECTION(op); - ret = setarrayitem(op, i, v); + ret = array_del_slice(arrayobject_CAST(op), i, i+1); Py_END_CRITICAL_SECTION(); return ret; } @@ -1277,7 +1486,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i); + selfi = getarrayitem((PyObject *)self, i, self->ob_item); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1325,7 +1534,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i); + selfi = getarrayitem((PyObject *)self, i, self->ob_item); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1348,7 +1557,8 @@ array_contains_lock_held(PyObject *op, PyObject *v) int cmp; for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { - PyObject *opi = getarrayitem(op, i); + arrayobject *ap = (arrayobject *)op; + PyObject *opi = getarrayitem(op, i, ap->ob_item); if (opi == NULL) return -1; cmp = PyObject_RichCompareBool(opi, v, Py_EQ); @@ -1387,7 +1597,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self,i); + selfi = getarrayitem((PyObject *)self, i, self->ob_item); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1433,7 +1643,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem((PyObject *)self, i); + v = getarrayitem((PyObject *)self, i, self->ob_item); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -1777,7 +1987,7 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) return NULL; for (i = 0; i < n; i++) { PyObject *v = PyList_GET_ITEM(list, i); - if ((*self->ob_descr->setitem)(self, + if ((*self->ob_descr->setitem)(self->ob_item, Py_SIZE(self) - n + i, v) != 0) { array_resize(self, old_size); return NULL; @@ -1810,7 +2020,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem((PyObject *)self, i); + PyObject *v = getarrayitem((PyObject *)self, i, self->ob_item); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -2565,22 +2775,13 @@ array_repr(PyObject *op) } static PyObject* -array_subscr_lock_held(PyObject *op, PyObject *item) +array_subscr_slice_lock_held(PyObject *op, PyObject *item) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *self = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(self)); - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i==-1 && PyErr_Occurred()) { - return NULL; - } - if (i < 0) - i += Py_SIZE(self); - return array_item(op, i); - } - else if (PySlice_Check(item)) { + if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; size_t cur; PyObject* result; @@ -2632,9 +2833,21 @@ array_subscr_lock_held(PyObject *op, PyObject *item) static PyObject * array_subscr(PyObject *op, PyObject *item) { + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i==-1 && PyErr_Occurred()) { + return NULL; + } + Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + if (i < 0) { + i += size; + } + return array_item(op, i); + } + PyObject *ret; Py_BEGIN_CRITICAL_SECTION(op); - ret = array_subscr_lock_held(op, item); + ret = array_subscr_slice_lock_held(op, item); Py_END_CRITICAL_SECTION(); return ret; } @@ -2674,7 +2887,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) slicelength = 1; } else - return (*self->ob_descr->setitem)(self, i, value); + return (*self->ob_descr->setitem)(self->ob_item, i, value); } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -2808,17 +3021,28 @@ static int array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) { int ret; - array_state* state = find_array_state_by_type(Py_TYPE(op)); - if (value != NULL && array_Check(value, state)) { - Py_BEGIN_CRITICAL_SECTION2(op, value); - ret = array_ass_subscr_lock_held(op, item, value); - Py_END_CRITICAL_SECTION2(); - } - else { - Py_BEGIN_CRITICAL_SECTION(op); - ret = array_ass_subscr_lock_held(op, item, value); - Py_END_CRITICAL_SECTION(); + + if (value != NULL) { + array_state* state = find_array_state_by_type(Py_TYPE(op)); + if (array_Check(value, state)) { + Py_BEGIN_CRITICAL_SECTION2(op, value); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION2(); + return ret; + } + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += Pyarrayobject_GET_SIZE(op); + return setarrayitem_maybe_locked(op, i, value); + } } + + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION(); return ret; } @@ -2937,6 +3161,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (len > 0 && !array_Check(initial, state)) { Py_ssize_t i; + arrayobject *ap = arrayobject_CAST(a); for (i = 0; i < len; i++) { PyObject *v = PySequence_GetItem(initial, i); @@ -2944,7 +3169,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) Py_DECREF(a); return NULL; } - if (setarrayitem(a, i, v) != 0) { + if (setarrayitem(a, i, v, ap->ob_item) != 0) { Py_DECREF(v); Py_DECREF(a); return NULL; @@ -2973,9 +3198,20 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (n > 0) { arrayobject *self = (arrayobject *)a; +#ifdef Py_GIL_DISABLED + self->ob_item = arrayarray_set(self->ob_item, + (char *)ustr, n, sizeof(wchar_t), false); + PyMem_Free(ustr); + if (self->ob_item == NULL) { + Py_DECREF(a); + PyErr_NoMemory(); + return NULL; + } +#else // self->ob_item may be NULL but it is safe. PyMem_Free(self->ob_item); self->ob_item = (char *)ustr; +#endif Py_SET_SIZE(self, n); self->allocated = n; } @@ -2989,9 +3225,20 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) } arrayobject *self = (arrayobject *)a; +#ifdef Py_GIL_DISABLED + self->ob_item = arrayarray_set(self->ob_item, + (char *)ustr, n, sizeof(Py_UCS4), false); + PyMem_Free(ustr); + if (self->ob_item == NULL) { + Py_DECREF(a); + PyErr_NoMemory(); + return NULL; + } +#else // self->ob_item may be NULL but it is safe. PyMem_Free(self->ob_item); self->ob_item = (char *)ustr; +#endif Py_SET_SIZE(self, n); self->allocated = n; } @@ -3223,7 +3470,6 @@ array_iter(PyObject *op) it->ao = (arrayobject*)Py_NewRef(ao); it->index = 0; // -1 indicates exhausted - it->getitem = ao->ob_descr->getitem; PyObject_GC_Track(it); return (PyObject *)it; } @@ -3237,23 +3483,13 @@ arrayiter_next(PyObject *op) if (index < 0) { return NULL; } - PyObject *ret; arrayobject *ao = it->ao; #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(it)); assert(PyObject_TypeCheck(it, state->ArrayIterType)); assert(array_Check(ao, state)); #endif - - Py_BEGIN_CRITICAL_SECTION(ao); - if (index < Py_SIZE(ao)) { - ret = (*it->getitem)(ao, index); - } - else { - ret = NULL; - } - Py_END_CRITICAL_SECTION(); - + PyObject *ret = getarrayitem_maybe_locked((PyObject *)ao, index); if (ret != NULL) { FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, index + 1); } From 512c4c74c5022eae0c416b992c56a63a805c2a05 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:08:06 +0000 Subject: [PATCH 03/32] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst diff --git a/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst b/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst new file mode 100644 index 00000000000000..0f2eb614e42374 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst @@ -0,0 +1 @@ +Make :mod:`array` module safe under :term:`free threading`. From 060100fb125c836643ac7c7a3432461184d50266 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 2 Mar 2025 16:36:24 -0500 Subject: [PATCH 04/32] ensure_shared_on_resize() in one more place --- Modules/arraymodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 41a5371acaf125..3ea1132a9f5aa4 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -250,6 +250,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) if (newsize == 0) { #ifdef Py_GIL_DISABLED + ensure_shared_on_resize(self); arrayarray_free(self->ob_item, _PyObject_GC_IS_SHARED(self)); #else arrayarray_free(self->ob_item, false); @@ -799,7 +800,6 @@ getarrayitem(PyObject *op, Py_ssize_t i, char *items) assert(array_Check(op, state)); #endif arrayobject *ap = arrayobject_CAST(op); - assert(valid_index(i, Py_SIZE(op))); return (*ap->ob_descr->getitem)(items, i); } @@ -864,7 +864,6 @@ setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v, char *items) assert(array_Check(op, state)); #endif arrayobject *ap = arrayobject_CAST(op); - assert(valid_index(i, Py_SIZE(op))); return (*ap->ob_descr->setitem)(items, i, v); } From d0b17c6868f8e270c3690ac47c72a3e924f8edbc Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 3 Mar 2025 07:04:41 -0500 Subject: [PATCH 05/32] fix stupid direct return out of critical section --- Modules/arraymodule.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3ea1132a9f5aa4..592d0ce784a7e4 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -814,10 +814,12 @@ getarrayitem_locked(PyObject *op, Py_ssize_t i) } #endif if (!valid_index(i, Py_SIZE(op))) { - return NULL; + ret = NULL; + } + else { + arrayobject *ap = (arrayobject *)op; + ret = getarrayitem(op, i, ap->ob_item); } - arrayobject *ap = (arrayobject *)op; - ret = getarrayitem(op, i, ap->ob_item); Py_END_CRITICAL_SECTION(); return ret; } @@ -844,6 +846,8 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) return NULL; } return getarrayitem(op, i, items); + /* Could check size again here to make sure it hasn't changed during get, + but not sure would add anything of value. */ } #else // Py_GIL_DISABLED From c17b787dc6871a5d17cfef3a30e12a4de9848875 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 3 Mar 2025 08:08:33 -0500 Subject: [PATCH 06/32] requested changes --- Modules/arraymodule.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 592d0ce784a7e4..67f8ca9229e84b 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -69,7 +69,7 @@ typedef struct { PyObject *str_iter; } array_state; -static inline Py_ssize_t Pyarrayobject_GET_SIZE(PyObject *op) { +static inline Py_ssize_t PyArray_GET_SIZE(PyObject *op) { arrayobject *ao = (arrayobject *)op; #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(ao)->ob_size)); @@ -77,7 +77,7 @@ static inline Py_ssize_t Pyarrayobject_GET_SIZE(PyObject *op) { return Py_SIZE(ao); #endif } -#define Pyarrayobject_GET_SIZE(op) Pyarrayobject_GET_SIZE(_PyObject_CAST(op)) +#define PyArray_GET_SIZE(op) PyArray_GET_SIZE(_PyObject_CAST(op)) /* Forward declaration. */ static PyObject *array_array_frombytes(PyObject *self, PyObject *bytes); @@ -832,7 +832,7 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { return getarrayitem_locked(op, i); } - Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + Py_ssize_t size = PyArray_GET_SIZE(op); if (!valid_index(i, size)) { return NULL; } @@ -901,12 +901,12 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { return setarrayitem_locked(op, i, v); } - Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + Py_ssize_t size = PyArray_GET_SIZE(op); if (!valid_index(i, size)) { goto error; } arrayobject *ap = (arrayobject *)op; - char *items = _Py_atomic_load_ptr(&ap->ob_item); + char *items = _Py_atomic_load_ptr_relaxed(&ap->ob_item); if (items == NULL) { goto error; } @@ -981,7 +981,7 @@ array_dealloc(PyObject *op) if (self->ob_exports > 0) { PyErr_SetString(PyExc_SystemError, "deallocated array object has exported buffers"); - PyErr_Print(); + PyErr_WriteUnraisable(NULL); } if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(op); @@ -1119,7 +1119,7 @@ static Py_ssize_t array_length(PyObject *op) { arrayobject *self = arrayobject_CAST(op); - return Pyarrayobject_GET_SIZE(self); + return PyArray_GET_SIZE(self); } @@ -2841,7 +2841,7 @@ array_subscr(PyObject *op, PyObject *item) if (i==-1 && PyErr_Occurred()) { return NULL; } - Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + Py_ssize_t size = PyArray_GET_SIZE(op); if (i < 0) { i += size; } @@ -3038,7 +3038,7 @@ array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) - i += Pyarrayobject_GET_SIZE(op); + i += PyArray_GET_SIZE(op); return setarrayitem_maybe_locked(op, i, value); } } @@ -3570,7 +3570,7 @@ array_arrayiterator___setstate__(arrayiterobject *self, PyObject *state) index = -1; } else { - Py_ssize_t size = Pyarrayobject_GET_SIZE(self->ao); + Py_ssize_t size = PyArray_GET_SIZE(self->ao); if (index > size) { index = size; /* iterator at end */ } From 4fd8383a558f316b62f3eb2a4009b5f0569371cf Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 3 Mar 2025 10:24:42 -0500 Subject: [PATCH 07/32] downgrade to _Py_atomic_load_ptr_relaxed in missed place --- Modules/arraymodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 67f8ca9229e84b..38967e2ed1bcd7 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -837,7 +837,7 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) return NULL; } arrayobject *ap = (arrayobject *)op; - char *items = _Py_atomic_load_ptr(&ap->ob_item); + char *items = _Py_atomic_load_ptr_relaxed(&ap->ob_item); if (items == NULL) { return NULL; } From d00f2b754a04def598977f7bfc5299ab436d1f4c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 4 Mar 2025 13:46:11 -0500 Subject: [PATCH 08/32] arraymodule linked statically --- Include/internal/pycore_pymem.h | 2 +- Modules/Setup.bootstrap.in | 3 +++ Modules/Setup.stdlib.in | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index bcd13d80cba975..5386d4c5f83031 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -117,7 +117,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str); extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. -PyAPI_FUNC(void) _PyMem_FreeDelayed(void *ptr); +extern void _PyMem_FreeDelayed(void *ptr); // Enqueue an object to be freed possibly after some delay #ifdef Py_GIL_DISABLED diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 4dcc0f55176d0e..8b9f3465fddb2e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -37,3 +37,6 @@ _symtable symtablemodule.c # for systems without $HOME env, used by site._getuserbase() @MODULE_PWD_TRUE@pwd pwdmodule.c + +# for optimization purposes +@MODULE_ARRAY_TRUE@array arraymodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 6bb05a06a3465d..17fa05703fd6f1 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -28,7 +28,6 @@ ############################################################################ # Modules that should always be present (POSIX and Windows): -@MODULE_ARRAY_TRUE@array arraymodule.c @MODULE__ASYNCIO_TRUE@_asyncio _asynciomodule.c @MODULE__BISECT_TRUE@_bisect _bisectmodule.c @MODULE__CONTEXTVARS_TRUE@_contextvars _contextvarsmodule.c From a3e60046124613ce76f1f9c270e7af2346134217 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 5 Mar 2025 12:07:39 -0500 Subject: [PATCH 09/32] cleanups --- Lib/test/test_array.py | 4 +- Modules/arraymodule.c | 490 ++++++++++++++++++++--------------------- 2 files changed, 243 insertions(+), 251 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index a7d2105b938598..41b52ea424faff 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1172,14 +1172,14 @@ def test_create_from_bytes(self): @support.cpython_only def test_sizeof_with_buffer(self): a = array.array(self.typecode, self.example) - basesize = support.calcvobjsize('Pn2Pi') + basesize = support.calcvobjsize('PnPi') buffer_size = a.buffer_info()[1] * a.itemsize support.check_sizeof(self, a, basesize + buffer_size) @support.cpython_only def test_sizeof_without_buffer(self): a = array.array(self.typecode) - basesize = support.calcvobjsize('Pn2Pi') + basesize = support.calcvobjsize('PnPi') support.check_sizeof(self, a, basesize) def test_initialize_with_unicode(self): diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 38967e2ed1bcd7..d24bba34a0312c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,10 +42,14 @@ struct arraydescr { int is_signed; }; +typedef struct { + Py_ssize_t allocated; + char items[]; +} arraydata; + typedef struct arrayobject { PyObject_VAR_HEAD - char *ob_item; - Py_ssize_t allocated; + arraydata *data; const struct arraydescr *ob_descr; PyObject *weakreflist; /* List of weak references */ Py_ssize_t ob_exports; /* Number of exported buffers */ @@ -69,7 +73,8 @@ typedef struct { PyObject *str_iter; } array_state; -static inline Py_ssize_t PyArray_GET_SIZE(PyObject *op) { +static inline +Py_ssize_t PyArray_GET_SIZE(PyObject *op) { arrayobject *ao = (arrayobject *)op; #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(ao)->ob_size)); @@ -144,91 +149,74 @@ enum machine_format_code { #define array_Check(op, state) PyObject_TypeCheck(op, state->ArrayType) -#ifdef Py_GIL_DISABLED -typedef struct { - Py_ssize_t allocated; - char ob_item[]; -} _PyArrayArray; -#endif +static inline char * +arraydata_safeitems(arraydata *data) +{ + return data == NULL ? NULL : data->items; +} -static char * -arrayarray_alloc(Py_ssize_t size, int itemsize) +static arraydata * +arraydata_alloc(Py_ssize_t size, int itemsize) { -#ifdef Py_GIL_DISABLED if (size > PY_SSIZE_T_MAX/itemsize - 1) { return NULL; } - _PyArrayArray *array = PyMem_Malloc(sizeof(_PyArrayArray) + size * itemsize); - if (array == NULL) { + arraydata *data = (arraydata *)PyMem_Malloc(sizeof(arraydata) + size * itemsize); + if (data == NULL) { return NULL; } - array->allocated = size; - return array->ob_item; -#else - return PyMem_Malloc(size * itemsize); -#endif + data->allocated = size; + return data; } static void -arrayarray_free(char *items, bool use_qsbr) +arraydata_free(arraydata *data, bool use_qsbr) { #ifdef Py_GIL_DISABLED - if (items == NULL) { - return; - } - _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); if (use_qsbr) { - _PyMem_FreeDelayed(array); + if (data != NULL) { + _PyMem_FreeDelayed(data); + } } else { - PyMem_Free(array); + PyMem_Free(data); } #else - PyMem_Free(items); + PyMem_Free(data); #endif } -#ifdef Py_GIL_DISABLED - -static Py_ssize_t -arrayarray_capacity(char *items) +static arraydata * +arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int itemsize, bool use_qsbr) { - _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); - return array->allocated; -} - -static char * -arrayarray_set(char *items, char *newitems, Py_ssize_t newsize, int itemsize, bool use_qsbr) -{ - arrayarray_free(items, use_qsbr); - items = arrayarray_alloc(newsize, itemsize); - if (items != NULL) { - memcpy(items, newitems, newsize * itemsize); + arraydata_free(data, use_qsbr); + data = arraydata_alloc(newsize, itemsize); + if (data != NULL) { + memcpy(data->items, newitems, newsize * itemsize); } - return items; + return data; } -static void -ensure_shared_on_resize(arrayobject *self) +#ifndef Py_GIL_DISABLED +static arraydata * +arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - // Ensure that the array is freed using QSBR if we are not the - // owning thread. - if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && - !_PyObject_GC_IS_SHARED(self)) - { - _PyObject_GC_SET_SHARED(self); + if (size > PY_SSIZE_T_MAX/itemsize - 1) { + return NULL; + } + data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); + if (data == NULL) { + return NULL; } + data->allocated = size; + return data; } - -# endif // Py_GIL_DISABLED +#endif static int array_resize(arrayobject *self, Py_ssize_t newsize) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - char *items; - size_t _new_size; if (self->ob_exports > 0 && newsize != Py_SIZE(self)) { PyErr_SetString(PyExc_BufferError, @@ -241,23 +229,33 @@ array_resize(arrayobject *self, Py_ssize_t newsize) current size, then proceed with the realloc() to shrink the array. */ - if (self->allocated >= newsize && - Py_SIZE(self) < newsize + 16 && - self->ob_item != NULL) { + arraydata *data = self->data; + + if (data != NULL && + data->allocated >= newsize && + Py_SIZE(self) < newsize + 16) { Py_SET_SIZE(self, newsize); return 0; } +#ifdef Py_GIL_DISABLED + // Ensure that the array is freed using QSBR if we are not the + // owning thread. + if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && + !_PyObject_GC_IS_SHARED(self)) + { + _PyObject_GC_SET_SHARED(self); + } +#endif + if (newsize == 0) { #ifdef Py_GIL_DISABLED - ensure_shared_on_resize(self); - arrayarray_free(self->ob_item, _PyObject_GC_IS_SHARED(self)); + arraydata_free(self->data, _PyObject_GC_IS_SHARED(self)); #else - arrayarray_free(self->ob_item, false); + arraydata_free(self->data, false); #endif - self->ob_item = NULL; + self->data = NULL; Py_SET_SIZE(self, 0); - FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, 0); return 0; } @@ -273,8 +271,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) * memory critical. */ - _new_size = (newsize >> 4) + (Py_SIZE(self) < 8 ? 3 : 7) + newsize; - items = self->ob_item; + size_t _new_size = (newsize >> 4) + (Py_SIZE(self) < 8 ? 3 : 7) + newsize; int itemsize = self->ob_descr->itemsize; /* XXX The following multiplication and division does not optimize away @@ -285,29 +282,27 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } #ifdef Py_GIL_DISABLED - ensure_shared_on_resize(self); - char *newitems = arrayarray_alloc(_new_size, itemsize); - if (newitems == NULL) { + arraydata *newdata = arraydata_alloc(_new_size, itemsize); + if (newdata == NULL) { PyErr_NoMemory(); return -1; } - if (items != NULL) { + if (data != NULL) { Py_ssize_t size = Py_SIZE(self); - memcpy(newitems, items, Py_MIN(size, newsize) * itemsize); - arrayarray_free(items, _PyObject_GC_IS_SHARED(self)); + memcpy(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); + arraydata_free(data, _PyObject_GC_IS_SHARED(self)); } - _Py_atomic_store_ptr_release(&self->ob_item, newitems); + _Py_atomic_store_ptr_release(&self->data, newdata); #else - PyMem_RESIZE(items, char, (_new_size * itemsize)); - if (items == NULL) { + data = arraydata_realloc(data, _new_size, itemsize); + if (data == NULL) { PyErr_NoMemory(); return -1; } - self->ob_item = items; + self->data = data; #endif Py_SET_SIZE(self, newsize); - FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, _new_size); return 0; } @@ -324,14 +319,14 @@ in bounds; that's the responsibility of the caller. ****************************************************************************/ static PyObject * -b_getitem(char *ob_item, Py_ssize_t i) +b_getitem(char *items, Py_ssize_t i) { - long x = ((signed char *)ob_item)[i]; + long x = ((signed char *)items)[i]; return PyLong_FromLong(x); } static int -b_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +b_setitem(char *items, Py_ssize_t i, PyObject *v) { short x; /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore @@ -350,37 +345,37 @@ b_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((char *)ob_item)[i] = (char)x; + ((char *)items)[i] = (char)x; return 0; } static PyObject * -BB_getitem(char *ob_item, Py_ssize_t i) +BB_getitem(char *items, Py_ssize_t i) { - long x = ((unsigned char *)ob_item)[i]; + long x = ((unsigned char *)items)[i]; return PyLong_FromLong(x); } static int -BB_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +BB_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; if (i >= 0) - ((unsigned char *)ob_item)[i] = x; + ((unsigned char *)items)[i] = x; return 0; } static PyObject * -u_getitem(char *ob_item, Py_ssize_t i) +u_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((wchar_t *) ob_item)[i]); + return PyUnicode_FromOrdinal(((wchar_t *) items)[i]); } static int -u_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +u_setitem(char *items, Py_ssize_t i, PyObject *v) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -411,19 +406,19 @@ u_setitem(char *ob_item, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { - ((wchar_t *)ob_item)[i] = w; + ((wchar_t *)items)[i] = w; } return 0; } static PyObject * -w_getitem(char *ob_item, Py_ssize_t i) +w_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((Py_UCS4 *) ob_item)[i]); + return PyUnicode_FromOrdinal(((Py_UCS4 *) items)[i]); } static int -w_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +w_setitem(char *items, Py_ssize_t i, PyObject *v) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -441,38 +436,38 @@ w_setitem(char *ob_item, Py_ssize_t i, PyObject *v) } if (i >= 0) { - ((Py_UCS4 *)ob_item)[i] = PyUnicode_READ_CHAR(v, 0); + ((Py_UCS4 *)items)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } static PyObject * -h_getitem(char *ob_item, Py_ssize_t i) +h_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((short *)ob_item)[i]); + return PyLong_FromLong((long) ((short *)items)[i]); } static int -h_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +h_setitem(char *items, Py_ssize_t i, PyObject *v) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; if (i >= 0) - ((short *)ob_item)[i] = x; + ((short *)items)[i] = x; return 0; } static PyObject * -HH_getitem(char *ob_item, Py_ssize_t i) +HH_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((unsigned short *)ob_item)[i]); + return PyLong_FromLong((long) ((unsigned short *)items)[i]); } static int -HH_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +HH_setitem(char *items, Py_ssize_t i, PyObject *v) { int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore @@ -490,37 +485,37 @@ HH_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((short *)ob_item)[i] = (short)x; + ((short *)items)[i] = (short)x; return 0; } static PyObject * -i_getitem(char *ob_item, Py_ssize_t i) +i_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((int *)ob_item)[i]); + return PyLong_FromLong((long) ((int *)items)[i]); } static int -i_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +i_setitem(char *items, Py_ssize_t i, PyObject *v) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; if (i >= 0) - ((int *)ob_item)[i] = x; + ((int *)items)[i] = x; return 0; } static PyObject * -II_getitem(char *ob_item, Py_ssize_t i) +II_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) ((unsigned int *)ob_item)[i]); + (unsigned long) ((unsigned int *)items)[i]); } static int -II_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +II_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -548,7 +543,7 @@ II_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned int *)ob_item)[i] = (unsigned int)x; + ((unsigned int *)items)[i] = (unsigned int)x; if (do_decref) { Py_DECREF(v); @@ -557,30 +552,30 @@ II_setitem(char *ob_item, Py_ssize_t i, PyObject *v) } static PyObject * -l_getitem(char *ob_item, Py_ssize_t i) +l_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong(((long *)ob_item)[i]); + return PyLong_FromLong(((long *)items)[i]); } static int -l_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +l_setitem(char *items, Py_ssize_t i, PyObject *v) { long x; if (!PyArg_Parse(v, "l;array item must be integer", &x)) return -1; if (i >= 0) - ((long *)ob_item)[i] = x; + ((long *)items)[i] = x; return 0; } static PyObject * -LL_getitem(char *ob_item, Py_ssize_t i) +LL_getitem(char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLong(((unsigned long *)ob_item)[i]); + return PyLong_FromUnsignedLong(((unsigned long *)items)[i]); } static int -LL_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +LL_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -600,7 +595,7 @@ LL_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned long *)ob_item)[i] = x; + ((unsigned long *)items)[i] = x; if (do_decref) { Py_DECREF(v); @@ -609,31 +604,31 @@ LL_setitem(char *ob_item, Py_ssize_t i, PyObject *v) } static PyObject * -q_getitem(char *ob_item, Py_ssize_t i) +q_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLongLong(((long long *)ob_item)[i]); + return PyLong_FromLongLong(((long long *)items)[i]); } static int -q_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +q_setitem(char *items, Py_ssize_t i, PyObject *v) { long long x; if (!PyArg_Parse(v, "L;array item must be integer", &x)) return -1; if (i >= 0) - ((long long *)ob_item)[i] = x; + ((long long *)items)[i] = x; return 0; } static PyObject * -QQ_getitem(char *ob_item, Py_ssize_t i) +QQ_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - ((unsigned long long *)ob_item)[i]); + ((unsigned long long *)items)[i]); } static int -QQ_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +QQ_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned long long x; int do_decref = 0; /* if nb_int was called */ @@ -653,7 +648,7 @@ QQ_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned long long *)ob_item)[i] = x; + ((unsigned long long *)items)[i] = x; if (do_decref) { Py_DECREF(v); @@ -662,36 +657,36 @@ QQ_setitem(char *ob_item, Py_ssize_t i, PyObject *v) } static PyObject * -f_getitem(char *ob_item, Py_ssize_t i) +f_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble((double) ((float *)ob_item)[i]); + return PyFloat_FromDouble((double) ((float *)items)[i]); } static int -f_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +f_setitem(char *items, Py_ssize_t i, PyObject *v) { float x; if (!PyArg_Parse(v, "f;array item must be float", &x)) return -1; if (i >= 0) - ((float *)ob_item)[i] = x; + ((float *)items)[i] = x; return 0; } static PyObject * -d_getitem(char *ob_item, Py_ssize_t i) +d_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble(((double *)ob_item)[i]); + return PyFloat_FromDouble(((double *)items)[i]); } static int -d_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +d_setitem(char *items, Py_ssize_t i, PyObject *v) { double x; if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; if (i >= 0) - ((double *)ob_item)[i] = x; + ((double *)items)[i] = x; return 0; } @@ -769,15 +764,14 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return NULL; } op->ob_descr = descr; - op->allocated = size; op->weakreflist = NULL; Py_SET_SIZE(op, size); if (size <= 0) { - op->ob_item = NULL; + op->data = NULL; } else { - op->ob_item = arrayarray_alloc(size, descr->itemsize); - if (op->ob_item == NULL) { + op->data = arraydata_alloc(size, descr->itemsize); + if (op->data == NULL) { Py_DECREF(op); return PyErr_NoMemory(); } @@ -792,14 +786,18 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) return (size_t) i < (size_t) limit; } -static PyObject * -getarrayitem(PyObject *op, Py_ssize_t i, char *items) +static inline PyObject * +getarrayitem(arrayobject *ap, Py_ssize_t i, char *items) { #ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(op)); - assert(array_Check(op, state)); + array_state *state = find_array_state_by_type(Py_TYPE(ap)); + assert(array_Check(ap, state)); +#ifdef Py_GIL_DISABLED + assert(valid_index(i, (_Py_CONTAINER_OF(items, arraydata, items))->allocated)); +#else + assert(valid_index(i, Py_SIZE(ap))); +#endif #endif - arrayobject *ap = arrayobject_CAST(op); return (*ap->ob_descr->getitem)(items, i); } @@ -818,7 +816,7 @@ getarrayitem_locked(PyObject *op, Py_ssize_t i) } else { arrayobject *ap = (arrayobject *)op; - ret = getarrayitem(op, i, ap->ob_item); + ret = getarrayitem(ap, i, ap->data->items); } Py_END_CRITICAL_SECTION(); return ret; @@ -837,17 +835,17 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) return NULL; } arrayobject *ap = (arrayobject *)op; - char *items = _Py_atomic_load_ptr_relaxed(&ap->ob_item); - if (items == NULL) { + arraydata *data = _Py_atomic_load_ptr_acquire(&ap->data); + if (data == NULL) { return NULL; } - Py_ssize_t cap = arrayarray_capacity(items); - if (!valid_index(i, cap)) { + if (!valid_index(i, data->allocated)) { return NULL; } - return getarrayitem(op, i, items); - /* Could check size again here to make sure it hasn't changed during get, - but not sure would add anything of value. */ + return getarrayitem(ap, i, data->items); + /* Could check size again here to make sure it hasn't changed but since + there isn't a well defined order between unsynchronized thread + operations there's no point. */ } #else // Py_GIL_DISABLED @@ -860,14 +858,20 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) #endif // Py_GIL_DISABLED -static int -setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v, char *items) +static inline int +setarrayitem(arrayobject *ap, Py_ssize_t i, PyObject *v, char *items) { #ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(op)); - assert(array_Check(op, state)); + array_state *state = find_array_state_by_type(Py_TYPE(ap)); + assert(array_Check(ap, state)); + if (items != NULL) { +#ifdef Py_GIL_DISABLED + assert(valid_index(i, (_Py_CONTAINER_OF(items, arraydata, items))->allocated)); +#else + assert(valid_index(i, Py_SIZE(ap))); +#endif + } #endif - arrayobject *ap = arrayobject_CAST(op); return (*ap->ob_descr->setitem)(items, i, v); } @@ -886,8 +890,8 @@ setarrayitem_locked(PyObject *op, Py_ssize_t i, PyObject *v) ret = -1; } else { - arrayobject *ap = (arrayobject *)op; - ret = setarrayitem(op, i, v, ap->ob_item); + arrayobject *ap = arrayobject_CAST(op); + ret = setarrayitem(ap, i, v, ap->data->items); } Py_END_CRITICAL_SECTION(); return ret; @@ -906,15 +910,16 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) goto error; } arrayobject *ap = (arrayobject *)op; - char *items = _Py_atomic_load_ptr_relaxed(&ap->ob_item); - if (items == NULL) { + arraydata *data = _Py_atomic_load_ptr_relaxed(&ap->data); + if (data == NULL) { goto error; } - Py_ssize_t cap = arrayarray_capacity(items); - if (!valid_index(i, cap)) { + if (!valid_index(i, data->allocated)) { goto error; } - return setarrayitem(op, i, v, items); + int ret = setarrayitem(ap, i, v, data->items); + _Py_atomic_fence_release(); + return ret; error: PyErr_SetString(PyExc_IndexError, "array index out of range"); @@ -941,12 +946,12 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) PyErr_BadInternalCall(); return -1; } - if ((*self->ob_descr->setitem)(self->ob_item, -1, v) < 0) + if (setarrayitem(self, -1, v, NULL) < 0) return -1; if (array_resize(self, n+1) == -1) return -1; - items = self->ob_item; + items = self->data->items; if (where < 0) { where += n; if (where < 0) @@ -959,7 +964,7 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) memmove(items + (where+1)*self->ob_descr->itemsize, items + where*self->ob_descr->itemsize, (n-where)*self->ob_descr->itemsize); - return (*self->ob_descr->setitem)(self->ob_item, where, v); + return setarrayitem(self, where, v, items); } /* Methods */ @@ -986,8 +991,8 @@ array_dealloc(PyObject *op) if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(op); } - if (self->ob_item != NULL) { - arrayarray_free(self->ob_item, false); + if (self->data != NULL) { + arraydata_free(self->data, false); } tp->tp_free(op); Py_DECREF(tp); @@ -1024,7 +1029,7 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Fast path: arrays with same types can have their buffers compared directly */ Py_ssize_t common_length = Py_MIN(Py_SIZE(va), Py_SIZE(wa)); - int result = va->ob_descr->compareitems(va->ob_item, wa->ob_item, + int result = va->ob_descr->compareitems(va->data->items, wa->data->items, common_length); if (result == 0) goto compare_sizes; @@ -1047,11 +1052,11 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(v, i, va->ob_item); + vi = getarrayitem(va, i, va->data->items); if (vi == NULL) { return NULL; } - wi = getarrayitem(w, i, wa->ob_item); + wi = getarrayitem(wa, i, wa->data->items); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -1155,7 +1160,7 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - memcpy(np->ob_item, a->ob_item + ilow * a->ob_descr->itemsize, + memcpy(np->data->items, a->data->items + ilow * a->ob_descr->itemsize, (ihigh-ilow) * a->ob_descr->itemsize); } return (PyObject *)np; @@ -1238,11 +1243,11 @@ array_concat_lock_held(PyObject *op, PyObject *bb) return NULL; } if (Py_SIZE(a) > 0) { - memcpy(np->ob_item, a->ob_item, Py_SIZE(a)*a->ob_descr->itemsize); + memcpy(np->data->items, a->data->items, Py_SIZE(a)*a->ob_descr->itemsize); } if (Py_SIZE(b) > 0) { - memcpy(np->ob_item + Py_SIZE(a)*a->ob_descr->itemsize, - b->ob_item, Py_SIZE(b)*b->ob_descr->itemsize); + memcpy(np->data->items + Py_SIZE(a)*a->ob_descr->itemsize, + b->data->items, Py_SIZE(b)*b->ob_descr->itemsize); } return (PyObject *)np; #undef b @@ -1280,7 +1285,7 @@ array_repeat_lock_held(PyObject *op, Py_ssize_t n) const Py_ssize_t oldbytes = array_length * a->ob_descr->itemsize; const Py_ssize_t newbytes = oldbytes * n; - _PyBytes_Repeat(np->ob_item, newbytes, a->ob_item, oldbytes); + _PyBytes_Repeat(np->data->items, newbytes, a->data->items, oldbytes); return (PyObject *)np; } @@ -1311,7 +1316,7 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) ihigh = ilow; else if (ihigh > Py_SIZE(a)) ihigh = Py_SIZE(a); - item = a->ob_item; + item = a->data->items; d = ihigh-ilow; /* Issue #4509: If the array has exported buffers and the slice assignment would change the size of the array, fail early to make @@ -1396,8 +1401,8 @@ array_do_extend_lock_held(array_state *state, arrayobject *self, PyObject *bb) if (array_resize(self, size) == -1) return -1; if (bbsize > 0) { - memcpy(self->ob_item + oldsize * self->ob_descr->itemsize, - b->ob_item, bbsize * b->ob_descr->itemsize); + memcpy(self->data->items + oldsize * self->ob_descr->itemsize, + b->data->items, bbsize * b->ob_descr->itemsize); } return 0; @@ -1452,7 +1457,7 @@ array_inplace_repeat_lock_held(PyObject *op, Py_ssize_t n) if (array_resize(self, n * array_size) == -1) return NULL; - _PyBytes_Repeat(self->ob_item, n*size, self->ob_item, size); + _PyBytes_Repeat(self->data->items, n*size, self->data->items, size); } return Py_NewRef(self); } @@ -1489,7 +1494,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i, self->ob_item); + selfi = getarrayitem(self, i, self->data->items); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1537,7 +1542,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i, self->ob_item); + selfi = getarrayitem(self, i, self->data->items); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1561,7 +1566,7 @@ array_contains_lock_held(PyObject *op, PyObject *v) for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { arrayobject *ap = (arrayobject *)op; - PyObject *opi = getarrayitem(op, i, ap->ob_item); + PyObject *opi = getarrayitem(ap, i, ap->data->items); if (opi == NULL) return -1; cmp = PyObject_RichCompareBool(opi, v, Py_EQ); @@ -1600,7 +1605,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i, self->ob_item); + selfi = getarrayitem(self, i, self->data->items); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1646,7 +1651,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem((PyObject *)self, i, self->ob_item); + v = getarrayitem(self, i, self->data->items); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -1717,7 +1722,7 @@ array_array_buffer_info_impl(arrayobject *self) if (!retval) return NULL; - v = PyLong_FromVoidPtr(self->ob_item); + v = PyLong_FromVoidPtr(arraydata_safeitems(self->data)); if (v == NULL) { Py_DECREF(retval); return NULL; @@ -1774,14 +1779,14 @@ array_array_byteswap_impl(arrayobject *self) case 1: break; case 2: - for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 2) { + for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 2) { char p0 = p[0]; p[0] = p[1]; p[1] = p0; } break; case 4: - for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 4) { + for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 4) { char p0 = p[0]; char p1 = p[1]; p[0] = p[3]; @@ -1791,7 +1796,7 @@ array_array_byteswap_impl(arrayobject *self) } break; case 8: - for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 8) { + for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 8) { char p0 = p[0]; char p1 = p[1]; char p2 = p[2]; @@ -1832,8 +1837,8 @@ array_array_reverse_impl(arrayobject *self) assert((size_t)itemsize <= sizeof(tmp)); if (Py_SIZE(self) > 1) { - for (p = self->ob_item, - q = self->ob_item + (Py_SIZE(self) - 1)*itemsize; + for (p = self->data->items, + q = self->data->items + (Py_SIZE(self) - 1)*itemsize; p < q; p += itemsize, q -= itemsize) { /* memory areas guaranteed disjoint, so memcpy @@ -1942,7 +1947,7 @@ array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) assert(state != NULL); for (i = 0; i < nblocks; i++) { - char* ptr = self->ob_item + i*BLOCKSIZE; + char *ptr = self->data->items + i*BLOCKSIZE; Py_ssize_t size = BLOCKSIZE; PyObject *bytes, *res; @@ -1990,8 +1995,7 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) return NULL; for (i = 0; i < n; i++) { PyObject *v = PyList_GET_ITEM(list, i); - if ((*self->ob_descr->setitem)(self->ob_item, - Py_SIZE(self) - n + i, v) != 0) { + if (setarrayitem(self, Py_SIZE(self) - n + i, v, self->data->items) != 0) { array_resize(self, old_size); return NULL; } @@ -2023,7 +2027,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem((PyObject *)self, i, self->ob_item); + PyObject *v = getarrayitem(self, i, self->data->items); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -2072,7 +2076,7 @@ array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) if (array_resize(self, old_size + n) == -1) { return NULL; } - memcpy(self->ob_item + old_size * itemsize, + memcpy(self->data->items + old_size * itemsize, buffer->buf, n * itemsize); } Py_RETURN_NONE; @@ -2090,7 +2094,7 @@ array_array_tobytes_impl(arrayobject *self) /*[clinic end generated code: output=87318e4edcdc2bb6 input=c4d44d5499d2320f]*/ { if (Py_SIZE(self) <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { - return PyBytes_FromStringAndSize(self->ob_item, + return PyBytes_FromStringAndSize(arraydata_safeitems(self->data), Py_SIZE(self) * self->ob_descr->itemsize); } else { return PyErr_NoMemory(); @@ -2135,7 +2139,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) // must not fail PyUnicode_AsWideChar( - ustr, ((wchar_t *)self->ob_item) + old_size, ustr_length); + ustr, ((wchar_t *)arraydata_safeitems(self->data)) + old_size, ustr_length); } } else { // typecode == 'w' @@ -2151,7 +2155,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) } // must not fail - Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)self->ob_item) + old_size, + Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)arraydata_safeitems(self->data)) + old_size, ustr_length, 0); assert(u != NULL); (void)u; // Suppress unused_variable warning. @@ -2182,11 +2186,11 @@ array_array_tounicode_impl(arrayobject *self) return NULL; } if (typecode == 'u') { - return PyUnicode_FromWideChar((wchar_t *) self->ob_item, Py_SIZE(self)); + return PyUnicode_FromWideChar((wchar_t *) arraydata_safeitems(self->data), Py_SIZE(self)); } else { // typecode == 'w' int byteorder = 0; // native byteorder - return PyUnicode_DecodeUTF32((const char *) self->ob_item, Py_SIZE(self) * 4, + return PyUnicode_DecodeUTF32((const char *) arraydata_safeitems(self->data), Py_SIZE(self) * 4, NULL, &byteorder); } } @@ -2202,8 +2206,10 @@ array_array___sizeof___impl(arrayobject *self) /*[clinic end generated code: output=d8e1c61ebbe3eaed input=805586565bf2b3c6]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated) - * (size_t)self->ob_descr->itemsize; + arraydata *data = FT_ATOMIC_LOAD_PTR_RELAXED(self->data); + if (data != NULL) { + res += (size_t)data->allocated * (size_t)self->ob_descr->itemsize; + } return PyLong_FromSize_t(res); } @@ -2805,8 +2811,8 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) slicelength, self->ob_descr); if (result == NULL) return NULL; - memcpy(((arrayobject *)result)->ob_item, - self->ob_item + start * itemsize, + memcpy(((arrayobject *)result)->data->items, + self->data->items + start * itemsize, slicelength * itemsize); return result; } @@ -2818,8 +2824,8 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) for (cur = start, i = 0; i < slicelength; cur += step, i++) { - memcpy(ar->ob_item + i*itemsize, - self->ob_item + cur*itemsize, + memcpy(ar->data->items + i*itemsize, + self->data->items + cur*itemsize, itemsize); } @@ -2890,7 +2896,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) slicelength = 1; } else - return (*self->ob_descr->setitem)(self->ob_item, i, value); + return setarrayitem(self, i, value, self->data->items); } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -2949,8 +2955,8 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (step == 1) { if (slicelength > needed) { - memmove(self->ob_item + (start + needed) * itemsize, - self->ob_item + stop * itemsize, + memmove(self->data->items + (start + needed) * itemsize, + self->data->items + stop * itemsize, (Py_SIZE(self) - stop) * itemsize); if (array_resize(self, Py_SIZE(self) + needed - slicelength) < 0) @@ -2960,13 +2966,13 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (array_resize(self, Py_SIZE(self) + needed - slicelength) < 0) return -1; - memmove(self->ob_item + (start + needed) * itemsize, - self->ob_item + stop * itemsize, + memmove(self->data->items + (start + needed) * itemsize, + self->data->items + stop * itemsize, (Py_SIZE(self) - start - needed) * itemsize); } if (needed > 0) - memcpy(self->ob_item + start * itemsize, - other->ob_item, needed * itemsize); + memcpy(self->data->items + start * itemsize, + other->data->items, needed * itemsize); return 0; } else if (needed == 0) { @@ -2985,14 +2991,14 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (cur + step >= (size_t)Py_SIZE(self)) lim = Py_SIZE(self) - cur - 1; - memmove(self->ob_item + (cur - i) * itemsize, - self->ob_item + (cur + 1) * itemsize, + memmove(self->data->items + (cur - i) * itemsize, + self->data->items + (cur + 1) * itemsize, lim * itemsize); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->ob_item + (cur-slicelength) * itemsize, - self->ob_item + cur * itemsize, + memmove(self->data->items + (cur-slicelength) * itemsize, + self->data->items + cur * itemsize, (Py_SIZE(self) - cur) * itemsize); } if (array_resize(self, Py_SIZE(self) - slicelength) < 0) @@ -3012,8 +3018,8 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) } for (cur = start, i = 0; i < slicelength; cur += step, i++) { - memcpy(self->ob_item + cur * itemsize, - other->ob_item + i * itemsize, + memcpy(self->data->items + cur * itemsize, + other->data->items + i * itemsize, itemsize); } return 0; @@ -3063,7 +3069,7 @@ array_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) return -1; } - view->buf = (void *)self->ob_item; + view->buf = (void *)arraydata_safeitems(self->data); view->obj = Py_NewRef(self); if (view->buf == NULL) view->buf = (void *)emptybuf; @@ -3172,7 +3178,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) Py_DECREF(a); return NULL; } - if (setarrayitem(a, i, v, ap->ob_item) != 0) { + if (setarrayitem(ap, i, v, ap->data->items) != 0) { Py_DECREF(v); Py_DECREF(a); return NULL; @@ -3201,22 +3207,15 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (n > 0) { arrayobject *self = (arrayobject *)a; -#ifdef Py_GIL_DISABLED - self->ob_item = arrayarray_set(self->ob_item, + self->data = arraydata_set_items(self->data, (char *)ustr, n, sizeof(wchar_t), false); PyMem_Free(ustr); - if (self->ob_item == NULL) { + if (self->data == NULL) { Py_DECREF(a); PyErr_NoMemory(); return NULL; } -#else - // self->ob_item may be NULL but it is safe. - PyMem_Free(self->ob_item); - self->ob_item = (char *)ustr; -#endif Py_SET_SIZE(self, n); - self->allocated = n; } } else { // c == 'w' @@ -3228,28 +3227,21 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) } arrayobject *self = (arrayobject *)a; -#ifdef Py_GIL_DISABLED - self->ob_item = arrayarray_set(self->ob_item, + self->data = arraydata_set_items(self->data, (char *)ustr, n, sizeof(Py_UCS4), false); PyMem_Free(ustr); - if (self->ob_item == NULL) { + if (self->data == NULL) { Py_DECREF(a); PyErr_NoMemory(); return NULL; } -#else - // self->ob_item may be NULL but it is safe. - PyMem_Free(self->ob_item); - self->ob_item = (char *)ustr; -#endif Py_SET_SIZE(self, n); - self->allocated = n; } } else if (initial != NULL && array_Check(initial, state) && len > 0) { arrayobject *self = (arrayobject *)a; arrayobject *other = (arrayobject *)initial; - memcpy(self->ob_item, other->ob_item, len * other->ob_descr->itemsize); + memcpy(self->data->items, other->data->items, len * other->ob_descr->itemsize); } if (it != NULL) { if (array_iter_extend((arrayobject *)a, it) == -1) { @@ -3608,7 +3600,7 @@ static PyType_Spec arrayiter_spec = { /*********************** Install Module **************************/ static int -array_traverse(PyObject *module, visitproc visit, void *arg) +arraymodule_traverse(PyObject *module, visitproc visit, void *arg) { array_state *state = get_array_state(module); Py_VISIT(state->ArrayType); @@ -3618,7 +3610,7 @@ array_traverse(PyObject *module, visitproc visit, void *arg) } static int -array_clear(PyObject *module) +arraymodule_clear(PyObject *module) { array_state *state = get_array_state(module); Py_CLEAR(state->ArrayType); @@ -3632,9 +3624,9 @@ array_clear(PyObject *module) } static void -array_free(void *module) +arraymodule_free(void *module) { - (void)array_clear((PyObject *)module); + (void)arraymodule_clear((PyObject *)module); } /* No functions in array module. */ @@ -3661,7 +3653,7 @@ do { \ } while (0) static int -array_modexec(PyObject *m) +arraymodule_modexec(PyObject *m) { array_state *state = get_array_state(m); char buffer[Py_ARRAY_LENGTH(descriptors)], *p; @@ -3716,7 +3708,7 @@ array_modexec(PyObject *m) } static PyModuleDef_Slot arrayslots[] = { - {Py_mod_exec, array_modexec}, + {Py_mod_exec, arraymodule_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} @@ -3730,9 +3722,9 @@ static struct PyModuleDef arraymodule = { .m_doc = module_doc, .m_methods = a_methods, .m_slots = arrayslots, - .m_traverse = array_traverse, - .m_clear = array_clear, - .m_free = array_free, + .m_traverse = arraymodule_traverse, + .m_clear = arraymodule_clear, + .m_free = arraymodule_free, }; From fb6212a500a785ff95a60d2966c75fba592f710a Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 5 Mar 2025 13:57:48 -0500 Subject: [PATCH 10/32] test and tsan stuff --- Lib/test/test_array.py | 94 ++++++++++++++++++++++++------------------ Modules/arraymodule.c | 11 ++--- 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 41b52ea424faff..a438ba0a9eb362 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -4,6 +4,7 @@ import collections.abc import io +import os import unittest from test import support from test.support import import_helper @@ -104,7 +105,10 @@ def test_empty(self): class ArrayReconstructorTest(unittest.TestCase): def setUp(self): - self.enterContext(warnings.catch_warnings()) + if (not support.Py_GIL_DISABLED or + any('"parallel_threads": null' in a for a in sys.argv) or + all('parallel_threads' not in a for a in sys.argv)): + self.enterContext(warnings.catch_warnings()) warnings.filterwarnings( "ignore", message="The 'u' type code is deprecated and " @@ -220,7 +224,10 @@ class BaseTest: # minitemsize: the minimum guaranteed itemsize def setUp(self): - self.enterContext(warnings.catch_warnings()) + if (not support.Py_GIL_DISABLED or + any('"parallel_threads": null' in a for a in sys.argv) or + all('parallel_threads' not in a for a in sys.argv)): + self.enterContext(warnings.catch_warnings()) warnings.filterwarnings( "ignore", message="The 'u' type code is deprecated and " @@ -472,54 +479,59 @@ def test_insert(self): def test_tofromfile(self): a = array.array(self.typecode, 2*self.example) self.assertRaises(TypeError, a.tofile) - os_helper.unlink(os_helper.TESTFN) - f = open(os_helper.TESTFN, 'wb') - try: - a.tofile(f) - f.close() - b = array.array(self.typecode) - f = open(os_helper.TESTFN, 'rb') - self.assertRaises(TypeError, b.fromfile) - b.fromfile(f, len(self.example)) - self.assertEqual(b, array.array(self.typecode, self.example)) - self.assertNotEqual(a, b) - self.assertRaises(EOFError, b.fromfile, f, len(self.example)+1) - self.assertEqual(a, b) - f.close() - finally: - if not f.closed: + with os_helper.temp_dir() as temp_dir: + temp_path = os.path.join(temp_dir, os_helper.TESTFN) + f = open(temp_path, 'wb') + try: + a.tofile(f) f.close() - os_helper.unlink(os_helper.TESTFN) + b = array.array(self.typecode) + f = open(temp_path, 'rb') + self.assertRaises(TypeError, b.fromfile) + b.fromfile(f, len(self.example)) + self.assertEqual(b, array.array(self.typecode, self.example)) + self.assertNotEqual(a, b) + self.assertRaises(EOFError, b.fromfile, f, len(self.example)+1) + self.assertEqual(a, b) + f.close() + finally: + if not f.closed: + f.close() + os_helper.unlink(temp_path) def test_fromfile_ioerror(self): # Issue #5395: Check if fromfile raises a proper OSError # instead of EOFError. a = array.array(self.typecode) - f = open(os_helper.TESTFN, 'wb') - try: - self.assertRaises(OSError, a.fromfile, f, len(self.example)) - finally: - f.close() - os_helper.unlink(os_helper.TESTFN) + with os_helper.temp_dir() as temp_dir: + temp_path = os.path.join(temp_dir, os_helper.TESTFN) + f = open(temp_path, 'wb') + try: + self.assertRaises(OSError, a.fromfile, f, len(self.example)) + finally: + f.close() + os_helper.unlink(temp_path) def test_filewrite(self): a = array.array(self.typecode, 2*self.example) - f = open(os_helper.TESTFN, 'wb') - try: - f.write(a) - f.close() - b = array.array(self.typecode) - f = open(os_helper.TESTFN, 'rb') - b.fromfile(f, len(self.example)) - self.assertEqual(b, array.array(self.typecode, self.example)) - self.assertNotEqual(a, b) - b.fromfile(f, len(self.example)) - self.assertEqual(a, b) - f.close() - finally: - if not f.closed: + with os_helper.temp_dir() as temp_dir: + temp_path = os.path.join(temp_dir, os_helper.TESTFN) + f = open(temp_path, 'wb') + try: + f.write(a) + f.close() + b = array.array(self.typecode) + f = open(temp_path, 'rb') + b.fromfile(f, len(self.example)) + self.assertEqual(b, array.array(self.typecode, self.example)) + self.assertNotEqual(a, b) + b.fromfile(f, len(self.example)) + self.assertEqual(a, b) f.close() - os_helper.unlink(os_helper.TESTFN) + finally: + if not f.closed: + f.close() + os_helper.unlink(temp_path) def test_tofromlist(self): a = array.array(self.typecode, 2*self.example) @@ -1201,6 +1213,7 @@ def test_obsolete_write_lock(self): a = array.array('B', b"") self.assertRaises(BufferError, _testcapi.getbuffer_with_null_view, a) + @unittest.skipIf(support.Py_GIL_DISABLED, 'not freed if GIL disabled') def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, array.array, (self.typecode,)) @@ -1255,6 +1268,7 @@ def test_issue17223(self): self.assertRaises(ValueError, a.tounicode) self.assertRaises(ValueError, str, a) + @unittest.skipIf(support.Py_GIL_DISABLED, 'warning stuff is not free-thread safe yet') def test_typecode_u_deprecation(self): with self.assertWarns(DeprecationWarning): array.array("u") diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index d24bba34a0312c..37e0d8f8a3787b 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -254,8 +254,8 @@ array_resize(arrayobject *self, Py_ssize_t newsize) #else arraydata_free(self->data, false); #endif - self->data = NULL; Py_SET_SIZE(self, 0); + FT_ATOMIC_STORE_PTR_RELAXED(self->data, NULL); return 0; } @@ -2629,10 +2629,11 @@ array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, array_state *state = get_array_state_by_class(cls); assert(state != NULL); - if (state->array_reconstructor == NULL) { - state->array_reconstructor = PyImport_ImportModuleAttrString( - "array", "_array_reconstructor"); - if (state->array_reconstructor == NULL) { + if (FT_ATOMIC_LOAD_PTR_RELAXED(state->array_reconstructor) == NULL) { + PyObject *array_reconstructor = PyImport_ImportModuleAttrString( + "array", "_array_reconstructor"); + FT_ATOMIC_STORE_PTR_RELAXED(state->array_reconstructor, array_reconstructor); + if (array_reconstructor == NULL) { return NULL; } } From 04a8f9dc9fe07f432ef206c90e660a0dfe74ac48 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 5 Mar 2025 18:51:44 -0500 Subject: [PATCH 11/32] getters and setters using atomics --- Lib/test/test_array.py | 161 ++++++++++++++++++++++++----------------- Modules/arraymodule.c | 142 ++++++++++++++++++++++-------------- 2 files changed, 184 insertions(+), 119 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index a438ba0a9eb362..8b759c5929eff8 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -105,10 +105,7 @@ def test_empty(self): class ArrayReconstructorTest(unittest.TestCase): def setUp(self): - if (not support.Py_GIL_DISABLED or - any('"parallel_threads": null' in a for a in sys.argv) or - all('parallel_threads' not in a for a in sys.argv)): - self.enterContext(warnings.catch_warnings()) + self.enterContext(warnings.catch_warnings()) warnings.filterwarnings( "ignore", message="The 'u' type code is deprecated and " @@ -224,10 +221,7 @@ class BaseTest: # minitemsize: the minimum guaranteed itemsize def setUp(self): - if (not support.Py_GIL_DISABLED or - any('"parallel_threads": null' in a for a in sys.argv) or - all('parallel_threads' not in a for a in sys.argv)): - self.enterContext(warnings.catch_warnings()) + self.enterContext(warnings.catch_warnings()) warnings.filterwarnings( "ignore", message="The 'u' type code is deprecated and " @@ -1696,6 +1690,29 @@ class FreeThreadingTest(unittest.TestCase): # Non-deterministic, but at least one of these things will fail if # array module is not free-thread safe. + def setUp(self): + self.enterContext(warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="The 'u' type code is deprecated and " + "will be removed in Python 3.16", + category=DeprecationWarning) + + def check(self, funcs, a=None, *args): + if a is None: + a = array.array('i', [1]) + + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier, a, *args)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + @unittest.skipUnless(support.Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() @@ -1889,67 +1906,81 @@ def iter_reduce(b, a, it): c = it.__reduce__() assert not c[1] or 0xdd not in c[1][0] - def check(funcs, a=None, *args): - if a is None: - a = array.array('i', [1]) - - barrier = threading.Barrier(len(funcs)) - threads = [] - - for func in funcs: - thread = threading.Thread(target=func, args=(barrier, a, *args)) - - threads.append(thread) - - with threading_helper.start_threads(threads): - pass - - check([pop1] * 10) - check([pop1] + [subscr0] * 10) - check([append1] * 10) - check([insert1] * 10) - check([pop1] + [index1] * 10) - check([pop1] + [contains1] * 10) - check([insert1] + [repeat2] * 10) - check([pop1] + [repr1] * 10) - check([inplace_repeat2] * 10) - check([byteswap] * 10) - check([insert1] + [clear] * 10) - check([pop1] + [count1] * 10) - check([remove1] * 10) - check([clear] + [copy] * 10, array.array('B', b'0' * 0x400000)) - check([pop1] + [reduce_ex2] * 10) - check([clear] + [reduce_ex3] * 10, array.array('B', b'0' * 0x400000)) - check([pop1] + [tobytes] * 10) - check([pop1] + [tolist] * 10) - check([clear, tounicode] * 10, array.array('w', 'a'*10000)) - check([clear, tofile] * 10, array.array('w', 'a'*10000)) - check([clear] + [extend] * 10) - check([clear] + [inplace_concat] * 10) - check([clear] + [concat] * 10, array.array('w', 'a'*10000)) - check([fromunicode] * 10, array.array('w', 'a')) - check([frombytes] * 10) - check([fromlist] * 10) - check([clear] + [richcmplhs] * 10, array.array('i', [1]*10000)) - check([clear] + [richcmprhs] * 10, array.array('i', [1]*10000)) - check([clear, ass0] * 10, array.array('i', [1]*10000)) # to test array_ass_item must disable Py_mp_ass_subscript - check([clear] + [new] * 10, array.array('w', 'a'*10000)) - check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) - check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) - check([clear] + [irepeat] * 10, array.array('B', b'0' * 0x40000)) - check([clear] + [iter_reduce] * 10, a := array.array('B', b'0' * 0x400), iter(a)) + self.check([pop1] * 10) + self.check([pop1] + [subscr0] * 10) + self.check([append1] * 10) + self.check([insert1] * 10) + self.check([pop1] + [index1] * 10) + self.check([pop1] + [contains1] * 10) + self.check([insert1] + [repeat2] * 10) + self.check([pop1] + [repr1] * 10) + self.check([inplace_repeat2] * 10) + self.check([byteswap] * 10) + self.check([insert1] + [clear] * 10) + self.check([pop1] + [count1] * 10) + self.check([remove1] * 10) + self.check([clear] + [copy] * 10, array.array('B', b'0' * 0x400000)) + self.check([pop1] + [reduce_ex2] * 10) + self.check([clear] + [reduce_ex3] * 10, array.array('B', b'0' * 0x400000)) + self.check([pop1] + [tobytes] * 10) + self.check([pop1] + [tolist] * 10) + self.check([clear, tounicode] * 10, array.array('w', 'a'*10000)) + self.check([clear, tofile] * 10, array.array('w', 'a'*10000)) + self.check([clear] + [extend] * 10) + self.check([clear] + [inplace_concat] * 10) + self.check([clear] + [concat] * 10, array.array('w', 'a'*10000)) + self.check([fromunicode] * 10, array.array('w', 'a')) + self.check([frombytes] * 10) + self.check([fromlist] * 10) + self.check([clear] + [richcmplhs] * 10, array.array('i', [1]*10000)) + self.check([clear] + [richcmprhs] * 10, array.array('i', [1]*10000)) + self.check([clear, ass0] * 10, array.array('i', [1]*10000)) # to test array_ass_item must disable Py_mp_ass_subscript + self.check([clear] + [new] * 10, array.array('w', 'a'*10000)) + self.check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) + self.check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) + self.check([clear] + [irepeat] * 10, array.array('B', b'0' * 0x40000)) + self.check([clear] + [iter_reduce] * 10, a := array.array('B', b'0' * 0x400), iter(a)) # make sure we handle non-self objects correctly - check([clear] + [newi] * 10, [2] * random.randint(0, 100)) - check([fromlistlclear] + [fromlistl] * 10, array.array('i', [1]), [2] * random.randint(0, 100)) - check([clear2] + [concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) - check([clear2] + [inplace_concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) - check([clear2] + [extend2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) - check([clear2] + [ass_subscr2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) - check([clear2] + [frombytes2] * 10, array.array('w', 'a'*10000), array.array('B', b'a'*10000)) + self.check([clear] + [newi] * 10, [2] * random.randint(0, 100)) + self.check([fromlistlclear] + [fromlistl] * 10, array.array('i', [1]), [2] * random.randint(0, 100)) + self.check([clear2] + [concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + self.check([clear2] + [inplace_concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + self.check([clear2] + [extend2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + self.check([clear2] + [ass_subscr2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + self.check([clear2] + [frombytes2] * 10, array.array('w', 'a'*10000), array.array('B', b'a'*10000)) # iterator stuff - check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) + self.check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) + + @unittest.skipUnless(support.check_sanitizer(thread=True), 'meant for tsan') + def test_free_threading_tsan(self): + def copy_back_and_forth(b, a, count): + b.wait() + for _ in range(count): + a[0] = a[1] + a[1] = a[0] + + def extend_range(b, a, count): + b.wait() + for _ in range(count): + a.extend(range(10)) + + def append_and_pop(b, a, count): + b.wait() + for _ in range(count): + a.append(1) + a.pop() + + for tc in typecodes: + if tc in 'uw': + a = array.array(tc, 'ab') + else: + a = array.array(tc, [0, 1]) + + self.check([copy_back_and_forth] * 10, a, 100) + self.check([append_and_pop] * 10, a, 100) + self.check([copy_back_and_forth] * 10 + [extend_range], a, 10) if __name__ == "__main__": diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 37e0d8f8a3787b..3cd7c645032ce7 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -321,8 +321,8 @@ in bounds; that's the responsibility of the caller. static PyObject * b_getitem(char *items, Py_ssize_t i) { - long x = ((signed char *)items)[i]; - return PyLong_FromLong(x); + return PyLong_FromLong( + (long) (signed char) FT_ATOMIC_LOAD_CHAR_RELAXED(items[i])); } static int @@ -332,8 +332,9 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore must use the next size up that is signed ('h') and manually do the overflow checking */ - if (!PyArg_Parse(v, "h;array item must be integer", &x)) + if (!PyArg_Parse(v, "h;array item must be integer", &x)) { return -1; + } else if (x < -128) { PyErr_SetString(PyExc_OverflowError, "signed char is less than minimum"); @@ -344,16 +345,17 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) "signed char is greater than maximum"); return -1; } - if (i >= 0) - ((char *)items)[i] = (char)x; + if (i >= 0) { + FT_ATOMIC_STORE_CHAR_RELAXED(items[i], (char) x); + } return 0; } static PyObject * BB_getitem(char *items, Py_ssize_t i) { - long x = ((unsigned char *)items)[i]; - return PyLong_FromLong(x); + return PyLong_FromLong( + (long) (unsigned char) FT_ATOMIC_LOAD_CHAR_RELAXED(items[i])); } static int @@ -361,17 +363,20 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ - if (!PyArg_Parse(v, "b;array item must be integer", &x)) + if (!PyArg_Parse(v, "b;array item must be integer", &x)) { return -1; - if (i >= 0) - ((unsigned char *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_CHAR_RELAXED(items[i], x); + } return 0; } static PyObject * u_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((wchar_t *) items)[i]); + return PyUnicode_FromOrdinal( + (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); } static int @@ -406,7 +411,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { - ((wchar_t *)items)[i] = w; + FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); } return 0; } @@ -414,7 +419,8 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * w_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((Py_UCS4 *) items)[i]); + return PyUnicode_FromOrdinal( + (Py_UCS4) FT_ATOMIC_LOAD_UINT32_RELAXED(((Py_UCS4 *) items)[i])); } static int @@ -436,7 +442,8 @@ w_setitem(char *items, Py_ssize_t i, PyObject *v) } if (i >= 0) { - ((Py_UCS4 *)items)[i] = PyUnicode_READ_CHAR(v, 0); + Py_UCS4 w = PyUnicode_READ_CHAR(v, 0); + FT_ATOMIC_STORE_UINT32_RELAXED(((Py_UCS4 *) items)[i], w); } return 0; } @@ -444,7 +451,8 @@ w_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * h_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((short *)items)[i]); + return PyLong_FromLong( + (long) FT_ATOMIC_LOAD_SHORT_RELAXED(((short *) items)[i])); } @@ -453,17 +461,20 @@ h_setitem(char *items, Py_ssize_t i, PyObject *v) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ - if (!PyArg_Parse(v, "h;array item must be integer", &x)) + if (!PyArg_Parse(v, "h;array item must be integer", &x)) { return -1; - if (i >= 0) - ((short *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_SHORT_RELAXED(((short *) items)[i], x); + } return 0; } static PyObject * HH_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((unsigned short *)items)[i]); + return PyLong_FromLong( + (long) (unsigned short) FT_ATOMIC_LOAD_SHORT_RELAXED(((short *) items)[i])); } static int @@ -472,8 +483,9 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore must use the next size up and manually do the overflow checking */ - if (!PyArg_Parse(v, "i;array item must be integer", &x)) + if (!PyArg_Parse(v, "i;array item must be integer", &x)) { return -1; + } else if (x < 0) { PyErr_SetString(PyExc_OverflowError, "unsigned short is less than minimum"); @@ -484,15 +496,17 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) "unsigned short is greater than maximum"); return -1; } - if (i >= 0) - ((short *)items)[i] = (short)x; + if (i >= 0) { + FT_ATOMIC_STORE_SHORT_RELAXED(((short *) items)[i], (unsigned short) x); + } return 0; } static PyObject * i_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((int *)items)[i]); + return PyLong_FromLong( + (long) FT_ATOMIC_LOAD_INT_RELAXED(((int *) items)[i])); } static int @@ -500,10 +514,12 @@ i_setitem(char *items, Py_ssize_t i, PyObject *v) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ - if (!PyArg_Parse(v, "i;array item must be integer", &x)) + if (!PyArg_Parse(v, "i;array item must be integer", &x)) { return -1; - if (i >= 0) - ((int *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_INT_RELAXED(((int *)items)[i], x); + } return 0; } @@ -511,7 +527,7 @@ static PyObject * II_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) ((unsigned int *)items)[i]); + (unsigned long) (unsigned int) FT_ATOMIC_LOAD_INT_RELAXED(((int *) items)[i])); } static int @@ -542,8 +558,9 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) - ((unsigned int *)items)[i] = (unsigned int)x; + if (i >= 0) { + FT_ATOMIC_STORE_INT_RELAXED(((int *) items)[i], (unsigned int) x); + } if (do_decref) { Py_DECREF(v); @@ -554,24 +571,28 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * l_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong(((long *)items)[i]); + return PyLong_FromLong( + FT_ATOMIC_LOAD_LONG_RELAXED(((long *) items)[i])); } static int l_setitem(char *items, Py_ssize_t i, PyObject *v) { long x; - if (!PyArg_Parse(v, "l;array item must be integer", &x)) + if (!PyArg_Parse(v, "l;array item must be integer", &x)) { return -1; - if (i >= 0) - ((long *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_LONG_RELAXED(((long *) items)[i], x); + } return 0; } static PyObject * LL_getitem(char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLong(((unsigned long *)items)[i]); + return PyLong_FromUnsignedLong( + (unsigned long) FT_ATOMIC_LOAD_LONG_RELAXED(((long *) items)[i])); } static int @@ -594,8 +615,9 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) - ((unsigned long *)items)[i] = x; + if (i >= 0) { + FT_ATOMIC_STORE_LONG_RELAXED(((long *) items)[i], x); + } if (do_decref) { Py_DECREF(v); @@ -606,17 +628,20 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * q_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLongLong(((long long *)items)[i]); + return PyLong_FromLongLong( + FT_ATOMIC_LOAD_LLONG_RELAXED(((long long *) items)[i])); } static int q_setitem(char *items, Py_ssize_t i, PyObject *v) { long long x; - if (!PyArg_Parse(v, "L;array item must be integer", &x)) + if (!PyArg_Parse(v, "L;array item must be integer", &x)) { return -1; - if (i >= 0) - ((long long *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_LLONG_RELAXED(((long long *) items)[i], x); + } return 0; } @@ -624,7 +649,7 @@ static PyObject * QQ_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - ((unsigned long long *)items)[i]); + (unsigned long long) FT_ATOMIC_LOAD_LLONG_RELAXED(((long long *) items)[i])); } static int @@ -647,8 +672,9 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) - ((unsigned long long *)items)[i] = x; + if (i >= 0) { + FT_ATOMIC_STORE_LLONG_RELAXED(((long long *) items)[i], x); + } if (do_decref) { Py_DECREF(v); @@ -659,34 +685,40 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * f_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble((double) ((float *)items)[i]); + return PyFloat_FromDouble( + (double) FT_ATOMIC_LOAD_FLOAT_RELAXED(((float *) items)[i])); } static int f_setitem(char *items, Py_ssize_t i, PyObject *v) { float x; - if (!PyArg_Parse(v, "f;array item must be float", &x)) + if (!PyArg_Parse(v, "f;array item must be float", &x)) { return -1; - if (i >= 0) - ((float *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_FLOAT_RELAXED(((float *) items)[i], x); + } return 0; } static PyObject * d_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble(((double *)items)[i]); + return PyFloat_FromDouble( + FT_ATOMIC_LOAD_DOUBLE_RELAXED(((double *) items)[i])); } static int d_setitem(char *items, Py_ssize_t i, PyObject *v) { double x; - if (!PyArg_Parse(v, "d;array item must be float", &x)) + if (!PyArg_Parse(v, "d;array item must be float", &x)) { return -1; - if (i >= 0) - ((double *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_DOUBLE_RELAXED(((double *) items)[i], x); + } return 0; } @@ -695,9 +727,11 @@ d_setitem(char *items, Py_ssize_t i, PyObject *v) code##_compareitems(const void *lhs, const void *rhs, Py_ssize_t length) \ { \ const type *a = lhs, *b = rhs; \ - for (Py_ssize_t i = 0; i < length; ++i) \ - if (a[i] != b[i]) \ + for (Py_ssize_t i = 0; i < length; ++i) { \ + if (a[i] != b[i]) { \ return a[i] < b[i] ? -1 : 1; \ + } \ + } \ return 0; \ } @@ -910,7 +944,7 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) goto error; } arrayobject *ap = (arrayobject *)op; - arraydata *data = _Py_atomic_load_ptr_relaxed(&ap->data); + arraydata *data = _Py_atomic_load_ptr_acquire(&ap->data); if (data == NULL) { goto error; } From 01423fd9f58b4be5fce171ce580443444b0c5799 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Mar 2025 08:26:59 -0500 Subject: [PATCH 12/32] fix 2 byte wchar_t --- Modules/arraymodule.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3cd7c645032ce7..bef93e7cdf6b7a 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -375,8 +375,13 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(char *items, Py_ssize_t i) { +#if WCHAR_MAX == 0x7FFFFFFF return PyUnicode_FromOrdinal( (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); +#else + return PyUnicode_FromOrdinal( + (wchar_t) FT_ATOMIC_LOAD_SHORT_RELAXED(((wchar_t *) items)[i])); +#endif } static int @@ -411,7 +416,11 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { +#if WCHAR_MAX == 0x7FFFFFFF FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); +#else + FT_ATOMIC_STORE_SHORT_RELAXED(((wchar_t *) items)[i], w); +#endif } return 0; } @@ -3235,6 +3244,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (c == 'u') { Py_ssize_t n; wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); + printf("... %ld, %ld\n", sizeof(wchar_t), n); if (ustr == NULL) { Py_DECREF(a); return NULL; From 99331dd257b892e842df4af5c61363f664ce6e30 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Mar 2025 08:42:27 -0500 Subject: [PATCH 13/32] remove debug printf --- Modules/arraymodule.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index bef93e7cdf6b7a..72eaf37bfb4a71 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3244,7 +3244,6 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (c == 'u') { Py_ssize_t n; wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); - printf("... %ld, %ld\n", sizeof(wchar_t), n); if (ustr == NULL) { Py_DECREF(a); return NULL; From 07c98cc6d5e9d7d3b2218121072b360ba9ae29d0 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Mar 2025 09:13:34 -0500 Subject: [PATCH 14/32] just a hunch... --- Modules/arraymodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 72eaf37bfb4a71..1fdb720b8cc9e1 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -375,7 +375,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(char *items, Py_ssize_t i) { -#if WCHAR_MAX == 0x7FFFFFFF +#if WCHAR_MAX > 0xFFFF return PyUnicode_FromOrdinal( (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); #else @@ -416,7 +416,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { -#if WCHAR_MAX == 0x7FFFFFFF +#if WCHAR_MAX > 0xFFFF FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); #else FT_ATOMIC_STORE_SHORT_RELAXED(((wchar_t *) items)[i], w); From cf3bbbb508205de823500867949e4cdf1c4a64a6 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Mar 2025 17:39:18 -0500 Subject: [PATCH 15/32] atomic "memcpy" --- Lib/test/libregrtest/tsan.py | 1 + Lib/test/test_array.py | 30 +++++++++++++++++----- Modules/arraymodule.c | 50 ++++++++++++++++++++++++++++++++---- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 77dde5ecb076e2..1fd92497ab954e 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -2,6 +2,7 @@ # chosen because they use threads and run in a reasonable amount of time. TSAN_TESTS = [ + 'test_array', # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). 'test_capi.test_mem', 'test_capi.test_pyatomic', diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 8b759c5929eff8..02feab8e119953 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1961,16 +1961,30 @@ def copy_back_and_forth(b, a, count): a[0] = a[1] a[1] = a[0] + def append_and_pop(b, a, count): + v = a[0] + b.wait() + for _ in range(count): + a.append(v) + a.pop() + + def copy_random(b, a, count): + end = len(a) - 1 + idxs = [(random.randint(0, end), random.randint(0, end)) for _ in range(count)] + b.wait() + for src, dst in idxs: + a[dst] = a[src] + def extend_range(b, a, count): b.wait() for _ in range(count): a.extend(range(10)) - def append_and_pop(b, a, count): + def extend_self(b, a, count): + c = a[:] b.wait() for _ in range(count): - a.append(1) - a.pop() + a.extend(c) for tc in typecodes: if tc in 'uw': @@ -1978,9 +1992,13 @@ def append_and_pop(b, a, count): else: a = array.array(tc, [0, 1]) - self.check([copy_back_and_forth] * 10, a, 100) - self.check([append_and_pop] * 10, a, 100) - self.check([copy_back_and_forth] * 10 + [extend_range], a, 10) + self.check([copy_back_and_forth] * 10, a, 100) + self.check([append_and_pop] * 10, a, 100) + self.check([copy_back_and_forth] * 10 + [extend_self], a, 100) + + if tc not in 'uw': + self.check([copy_back_and_forth] * 10 + [extend_range], a, 100) + self.check([copy_random] * 10, a * 5, 100) if __name__ == "__main__": diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 1fdb720b8cc9e1..3b4ecf128c4340 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -197,7 +197,44 @@ arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int ite return data; } +#ifdef Py_GIL_DISABLED + +static void +atomic_itemcpy(void *dest, const void *src, size_t n, int itemsize) +{ + if (itemsize == 1) { + for (char *d = (char *) dest, *end = d + n, *s = (char *) src; + d < end; d++, s++) { + _Py_atomic_store_char_relaxed(d, _Py_atomic_load_char_relaxed(s)); + } + } + else if (itemsize == 2) { + for (short *d = (short *) dest, *end = d + n, *s = (short *) src; + d < end; d++, s++) { + _Py_atomic_store_short_relaxed(d, _Py_atomic_load_short_relaxed(s)); + } + } + else if (itemsize == 4) { + for (PY_UINT32_T *d = (PY_UINT32_T *) dest, *end = d + n, *s = (PY_UINT32_T *) src; + d < end; d++, s++) { + _Py_atomic_store_uint32_relaxed(d, _Py_atomic_load_uint32_relaxed(s)); + } + } + else if (itemsize == 8) { + for (PY_UINT64_T *d = (PY_UINT64_T *) dest, *end = d + n, *s = (PY_UINT64_T *) src; + d < end; d++, s++) { + _Py_atomic_store_uint64_relaxed(d, _Py_atomic_load_uint64_relaxed(s)); + } + } + else { + assert(false); + } +} + +#endif + #ifndef Py_GIL_DISABLED + static arraydata * arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) { @@ -211,6 +248,7 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) data->allocated = size; return data; } + #endif static int @@ -289,7 +327,11 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } if (data != NULL) { Py_ssize_t size = Py_SIZE(self); +#ifdef Py_GIL_DISABLED + atomic_itemcpy(newdata->items, data->items, Py_MIN(size, newsize), itemsize); +#else memcpy(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); +#endif arraydata_free(data, _PyObject_GC_IS_SHARED(self)); } _Py_atomic_store_ptr_release(&self->data, newdata); @@ -375,7 +417,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(char *items, Py_ssize_t i) { -#if WCHAR_MAX > 0xFFFF +#if SIZEOF_WCHAR_T > 2 return PyUnicode_FromOrdinal( (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); #else @@ -416,7 +458,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { -#if WCHAR_MAX > 0xFFFF +#if SIZEOF_WCHAR_T > 2 FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); #else FT_ATOMIC_STORE_SHORT_RELAXED(((wchar_t *) items)[i], w); @@ -960,9 +1002,7 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) if (!valid_index(i, data->allocated)) { goto error; } - int ret = setarrayitem(ap, i, v, data->items); - _Py_atomic_fence_release(); - return ret; + return setarrayitem(ap, i, v, data->items); error: PyErr_SetString(PyExc_IndexError, "array index out of range"); From 51135a497933e4ac86652e9365a7b6dcda5d51b7 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 7 Mar 2025 02:15:43 -0500 Subject: [PATCH 16/32] misc --- Modules/arraymodule.c | 44 +++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3b4ecf128c4340..c83da9b35a48cc 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -197,6 +197,24 @@ arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int ite return data; } +#ifndef Py_GIL_DISABLED + +static arraydata * +arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) +{ + if (size > PY_SSIZE_T_MAX/itemsize - 1) { + return NULL; + } + data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); + if (data == NULL) { + return NULL; + } + data->allocated = size; + return data; +} + +#endif + #ifdef Py_GIL_DISABLED static void @@ -205,25 +223,25 @@ atomic_itemcpy(void *dest, const void *src, size_t n, int itemsize) if (itemsize == 1) { for (char *d = (char *) dest, *end = d + n, *s = (char *) src; d < end; d++, s++) { - _Py_atomic_store_char_relaxed(d, _Py_atomic_load_char_relaxed(s)); + *d = _Py_atomic_load_char_relaxed(s); } } else if (itemsize == 2) { for (short *d = (short *) dest, *end = d + n, *s = (short *) src; d < end; d++, s++) { - _Py_atomic_store_short_relaxed(d, _Py_atomic_load_short_relaxed(s)); + *d = _Py_atomic_load_short_relaxed(s); } } else if (itemsize == 4) { for (PY_UINT32_T *d = (PY_UINT32_T *) dest, *end = d + n, *s = (PY_UINT32_T *) src; d < end; d++, s++) { - _Py_atomic_store_uint32_relaxed(d, _Py_atomic_load_uint32_relaxed(s)); + *d = (PY_UINT32_T) _Py_atomic_load_uint32_relaxed(s); } } else if (itemsize == 8) { for (PY_UINT64_T *d = (PY_UINT64_T *) dest, *end = d + n, *s = (PY_UINT64_T *) src; d < end; d++, s++) { - _Py_atomic_store_uint64_relaxed(d, _Py_atomic_load_uint64_relaxed(s)); + *d = (PY_UINT64_T) _Py_atomic_load_uint64_relaxed(s); } } else { @@ -233,24 +251,6 @@ atomic_itemcpy(void *dest, const void *src, size_t n, int itemsize) #endif -#ifndef Py_GIL_DISABLED - -static arraydata * -arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) -{ - if (size > PY_SSIZE_T_MAX/itemsize - 1) { - return NULL; - } - data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); - if (data == NULL) { - return NULL; - } - data->allocated = size; - return data; -} - -#endif - static int array_resize(arrayobject *self, Py_ssize_t newsize) { From fff827e0525e7f9cd3d414ed8cd848e0283795f2 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 8 Mar 2025 17:29:40 -0500 Subject: [PATCH 17/32] use proper type FT_ macros --- Lib/test/test_array.py | 2 ++ Modules/arraymodule.c | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 02feab8e119953..bb2203de627f52 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1954,6 +1954,8 @@ def iter_reduce(b, a, it): self.check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) @unittest.skipUnless(support.check_sanitizer(thread=True), 'meant for tsan') + @threading_helper.reap_threads + @threading_helper.requires_working_threading() def test_free_threading_tsan(self): def copy_back_and_forth(b, a, count): b.wait() diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index c83da9b35a48cc..2f7444d063b8b0 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -397,7 +397,7 @@ static PyObject * BB_getitem(char *items, Py_ssize_t i) { return PyLong_FromLong( - (long) (unsigned char) FT_ATOMIC_LOAD_CHAR_RELAXED(items[i])); + (long) FT_ATOMIC_LOAD_UCHAR_RELAXED(((unsigned char *) items)[i])); } static int @@ -409,7 +409,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_CHAR_RELAXED(items[i], x); + FT_ATOMIC_STORE_UCHAR_RELAXED(((unsigned char *) items)[i], x); } return 0; } @@ -525,7 +525,7 @@ static PyObject * HH_getitem(char *items, Py_ssize_t i) { return PyLong_FromLong( - (long) (unsigned short) FT_ATOMIC_LOAD_SHORT_RELAXED(((short *) items)[i])); + (long) FT_ATOMIC_LOAD_USHORT_RELAXED(((unsigned short *) items)[i])); } static int @@ -548,7 +548,7 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_SHORT_RELAXED(((short *) items)[i], (unsigned short) x); + FT_ATOMIC_STORE_USHORT_RELAXED(((unsigned short *) items)[i], (unsigned short) x); } return 0; } @@ -578,7 +578,7 @@ static PyObject * II_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) (unsigned int) FT_ATOMIC_LOAD_INT_RELAXED(((int *) items)[i])); + (unsigned long) FT_ATOMIC_LOAD_UINT_RELAXED(((unsigned int *) items)[i])); } static int @@ -610,7 +610,7 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_INT_RELAXED(((int *) items)[i], (unsigned int) x); + FT_ATOMIC_STORE_UINT_RELAXED(((unsigned int *) items)[i], (unsigned int) x); } if (do_decref) { @@ -643,7 +643,7 @@ static PyObject * LL_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) FT_ATOMIC_LOAD_LONG_RELAXED(((long *) items)[i])); + FT_ATOMIC_LOAD_ULONG_RELAXED(((unsigned long *) items)[i])); } static int @@ -667,7 +667,7 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_LONG_RELAXED(((long *) items)[i], x); + FT_ATOMIC_STORE_ULONG_RELAXED(((unsigned long *) items)[i], x); } if (do_decref) { @@ -700,7 +700,7 @@ static PyObject * QQ_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - (unsigned long long) FT_ATOMIC_LOAD_LLONG_RELAXED(((long long *) items)[i])); + FT_ATOMIC_LOAD_ULLONG_RELAXED(((unsigned long long *) items)[i])); } static int @@ -724,7 +724,7 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_LLONG_RELAXED(((long long *) items)[i], x); + FT_ATOMIC_STORE_ULLONG_RELAXED(((unsigned long long *) items)[i], x); } if (do_decref) { From 12f0ff64afe5584982c3518bdf199607595aacd7 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 10 Mar 2025 19:18:31 -0400 Subject: [PATCH 18/32] atomic aggregate _Py_atomic_source_memcpy_relaxed() --- Modules/arraymodule.c | 149 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 27 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2f7444d063b8b0..d348d313c8c798 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -217,38 +217,136 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #ifdef Py_GIL_DISABLED +// This really doesn't belong here, for show at the moment. static void -atomic_itemcpy(void *dest, const void *src, size_t n, int itemsize) +_Py_atomic_source_memcpy_relaxed(void *dest, void *src, size_t n) { - if (itemsize == 1) { - for (char *d = (char *) dest, *end = d + n, *s = (char *) src; - d < end; d++, s++) { - *d = _Py_atomic_load_char_relaxed(s); + int diff = (int)((uintptr_t)dest ^ (uintptr_t)src); + + // the first half is needed to deal with misalignment + + if (diff & 1) { // dest and src not word aligned with each other + for (void *end = (char *)dest + n; dest < end; + dest = (char *)dest + 1, src = (char *)src + 1) { + *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); } + + return; } - else if (itemsize == 2) { - for (short *d = (short *) dest, *end = d + n, *s = (short *) src; - d < end; d++, s++) { - *d = _Py_atomic_load_short_relaxed(s); + + if ((uintptr_t)dest & 1) { // dest and src not word aligned in memory + if (n) { + *(char *)dest = _Py_atomic_load_char_relaxed((char *)src); + dest = (char *)dest + 1; + src = (char *)src + 1; + n -= 1; + } + + if (!n) { + return; } } - else if (itemsize == 4) { - for (PY_UINT32_T *d = (PY_UINT32_T *) dest, *end = d + n, *s = (PY_UINT32_T *) src; - d < end; d++, s++) { - *d = (PY_UINT32_T) _Py_atomic_load_uint32_relaxed(s); + + if (diff & 2) { // dest and src not dword aligned with each other + size_t n2 = n / 2; + + for (void *end = (short *)dest + n2; dest < end; + dest = (short *)dest + 1, src = (short *)src + 1) { + *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); + } + + if (n & 1) { + *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); } + + return; } - else if (itemsize == 8) { - for (PY_UINT64_T *d = (PY_UINT64_T *) dest, *end = d + n, *s = (PY_UINT64_T *) src; - d < end; d++, s++) { - *d = (PY_UINT64_T) _Py_atomic_load_uint64_relaxed(s); + + if ((uintptr_t)dest & 2) { // dest and src not dword aligned in memory + if (n >= 2) { + *(short *)dest = _Py_atomic_load_short_relaxed((short *)src); + dest = (short *)dest + 1; + src = (short *)src + 1; + n -= 2; + } + + if (!n) { + return; } } - else { - assert(false); + + if (diff & 4) { // dest and src not qword aligned with each other + size_t n4 = n / 4; + + for (void *end = (PY_UINT32_T *)dest + n4; dest < end; + dest = (PY_UINT32_T *)dest + 1, src = (PY_UINT32_T *)src + 1) { + *((PY_UINT32_T *)dest) = (PY_UINT32_T)_Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); + } + + if (n & 2) { + *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); + dest = (short *)dest + 1; + src = (short *)src + 1; + } + + if (n & 1) { + *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); + } + + return; + } + + if ((uintptr_t)dest & 4) { // dest and src not qword aligned in memory + if (n >= 4) { + *(PY_UINT32_T *)dest = _Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); + dest = (PY_UINT32_T *)dest + 1; + src = (PY_UINT32_T *)src + 1; + n -= 4; + } + + if (!n) { + return; + } + } + + // the second half is aligned copy + + size_t n8 = n / 8; + + if (n8) { + for (void *end = (PY_UINT64_T *)dest + n8; dest < end; + dest = (PY_UINT64_T *)dest + 1, src = (PY_UINT64_T *)src + 1) { + *((PY_UINT64_T *)dest) = (PY_UINT64_T)_Py_atomic_load_uint64_relaxed((PY_UINT64_T *)src); + } + + n -= n8 * 8; + } + + if (n & 4) { + *((PY_UINT32_T *)dest) = (PY_UINT32_T)_Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); + dest = (PY_UINT32_T *)dest + 1; + src = (PY_UINT32_T *)src + 1; + } + + if (n & 2) { + *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); + dest = (short *)dest + 1; + src = (short *)src + 1; + } + + if (n & 1) { + *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); } } +#define FT_ATOMIC_SOURCE_MEMCPY_RELAXED(dest, src, n) \ + _Py_atomic_source_memcpy_relaxed((dest), (src), (n)) + +#else + +#define FT_ATOMIC_SOURCE_MEMCPY_RELAXED(dest, src, n) \ + memcpy((dest), (src), (n)) + #endif static int @@ -327,11 +425,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } if (data != NULL) { Py_ssize_t size = Py_SIZE(self); -#ifdef Py_GIL_DISABLED - atomic_itemcpy(newdata->items, data->items, Py_MIN(size, newsize), itemsize); -#else - memcpy(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); -#endif + FT_ATOMIC_SOURCE_MEMCPY_RELAXED(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); arraydata_free(data, _PyObject_GC_IS_SHARED(self)); } _Py_atomic_store_ptr_release(&self->data, newdata); @@ -1243,8 +1337,9 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - memcpy(np->data->items, a->data->items + ilow * a->ob_descr->itemsize, - (ihigh-ilow) * a->ob_descr->itemsize); + FT_ATOMIC_SOURCE_MEMCPY_RELAXED( + np->data->items, a->data->items + ilow * a->ob_descr->itemsize, + (ihigh-ilow) * a->ob_descr->itemsize); } return (PyObject *)np; } @@ -2895,7 +2990,7 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) slicelength, self->ob_descr); if (result == NULL) return NULL; - memcpy(((arrayobject *)result)->data->items, + FT_ATOMIC_SOURCE_MEMCPY_RELAXED(((arrayobject *)result)->data->items, self->data->items + start * itemsize, slicelength * itemsize); return result; From 94ef41723ff1fb2288cb118bc900f5ef87a78255 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 11 Mar 2025 12:54:32 -0400 Subject: [PATCH 19/32] remove atomic memcpy --- Lib/test/test_array.py | 25 ++++---- Modules/arraymodule.c | 141 +---------------------------------------- 2 files changed, 16 insertions(+), 150 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index bb2203de627f52..5ad36c799e92c6 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1977,16 +1977,17 @@ def copy_random(b, a, count): for src, dst in idxs: a[dst] = a[src] - def extend_range(b, a, count): - b.wait() - for _ in range(count): - a.extend(range(10)) - - def extend_self(b, a, count): - c = a[:] - b.wait() - for _ in range(count): - a.extend(c) + # comment these back in if resize is made atomically quiet + # def extend_range(b, a, count): + # b.wait() + # for _ in range(count): + # a.extend(range(10)) + + # def extend_self(b, a, count): + # c = a[:] + # b.wait() + # for _ in range(count): + # a.extend(c) for tc in typecodes: if tc in 'uw': @@ -1996,10 +1997,10 @@ def extend_self(b, a, count): self.check([copy_back_and_forth] * 10, a, 100) self.check([append_and_pop] * 10, a, 100) - self.check([copy_back_and_forth] * 10 + [extend_self], a, 100) + # self.check([copy_back_and_forth] * 10 + [extend_self], a, 100) # comment this in if resize is made atomically quiet if tc not in 'uw': - self.check([copy_back_and_forth] * 10 + [extend_range], a, 100) + # self.check([copy_back_and_forth] * 10 + [extend_range], a, 100) # comment this in if resize is made atomically quiet self.check([copy_random] * 10, a * 5, 100) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index d348d313c8c798..966999789d6fb1 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -215,140 +215,6 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #endif -#ifdef Py_GIL_DISABLED - -// This really doesn't belong here, for show at the moment. -static void -_Py_atomic_source_memcpy_relaxed(void *dest, void *src, size_t n) -{ - int diff = (int)((uintptr_t)dest ^ (uintptr_t)src); - - // the first half is needed to deal with misalignment - - if (diff & 1) { // dest and src not word aligned with each other - for (void *end = (char *)dest + n; dest < end; - dest = (char *)dest + 1, src = (char *)src + 1) { - *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); - } - - return; - } - - if ((uintptr_t)dest & 1) { // dest and src not word aligned in memory - if (n) { - *(char *)dest = _Py_atomic_load_char_relaxed((char *)src); - dest = (char *)dest + 1; - src = (char *)src + 1; - n -= 1; - } - - if (!n) { - return; - } - } - - if (diff & 2) { // dest and src not dword aligned with each other - size_t n2 = n / 2; - - for (void *end = (short *)dest + n2; dest < end; - dest = (short *)dest + 1, src = (short *)src + 1) { - *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); - } - - if (n & 1) { - *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); - } - - return; - } - - if ((uintptr_t)dest & 2) { // dest and src not dword aligned in memory - if (n >= 2) { - *(short *)dest = _Py_atomic_load_short_relaxed((short *)src); - dest = (short *)dest + 1; - src = (short *)src + 1; - n -= 2; - } - - if (!n) { - return; - } - } - - if (diff & 4) { // dest and src not qword aligned with each other - size_t n4 = n / 4; - - for (void *end = (PY_UINT32_T *)dest + n4; dest < end; - dest = (PY_UINT32_T *)dest + 1, src = (PY_UINT32_T *)src + 1) { - *((PY_UINT32_T *)dest) = (PY_UINT32_T)_Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); - } - - if (n & 2) { - *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); - dest = (short *)dest + 1; - src = (short *)src + 1; - } - - if (n & 1) { - *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); - } - - return; - } - - if ((uintptr_t)dest & 4) { // dest and src not qword aligned in memory - if (n >= 4) { - *(PY_UINT32_T *)dest = _Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); - dest = (PY_UINT32_T *)dest + 1; - src = (PY_UINT32_T *)src + 1; - n -= 4; - } - - if (!n) { - return; - } - } - - // the second half is aligned copy - - size_t n8 = n / 8; - - if (n8) { - for (void *end = (PY_UINT64_T *)dest + n8; dest < end; - dest = (PY_UINT64_T *)dest + 1, src = (PY_UINT64_T *)src + 1) { - *((PY_UINT64_T *)dest) = (PY_UINT64_T)_Py_atomic_load_uint64_relaxed((PY_UINT64_T *)src); - } - - n -= n8 * 8; - } - - if (n & 4) { - *((PY_UINT32_T *)dest) = (PY_UINT32_T)_Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); - dest = (PY_UINT32_T *)dest + 1; - src = (PY_UINT32_T *)src + 1; - } - - if (n & 2) { - *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); - dest = (short *)dest + 1; - src = (short *)src + 1; - } - - if (n & 1) { - *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); - } -} - -#define FT_ATOMIC_SOURCE_MEMCPY_RELAXED(dest, src, n) \ - _Py_atomic_source_memcpy_relaxed((dest), (src), (n)) - -#else - -#define FT_ATOMIC_SOURCE_MEMCPY_RELAXED(dest, src, n) \ - memcpy((dest), (src), (n)) - -#endif - static int array_resize(arrayobject *self, Py_ssize_t newsize) { @@ -425,7 +291,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } if (data != NULL) { Py_ssize_t size = Py_SIZE(self); - FT_ATOMIC_SOURCE_MEMCPY_RELAXED(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); + memcpy(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); arraydata_free(data, _PyObject_GC_IS_SHARED(self)); } _Py_atomic_store_ptr_release(&self->data, newdata); @@ -1337,8 +1203,7 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - FT_ATOMIC_SOURCE_MEMCPY_RELAXED( - np->data->items, a->data->items + ilow * a->ob_descr->itemsize, + memcpy(np->data->items, a->data->items + ilow * a->ob_descr->itemsize, (ihigh-ilow) * a->ob_descr->itemsize); } return (PyObject *)np; @@ -2990,7 +2855,7 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) slicelength, self->ob_descr); if (result == NULL) return NULL; - FT_ATOMIC_SOURCE_MEMCPY_RELAXED(((arrayobject *)result)->data->items, + memcpy(((arrayobject *)result)->data->items, self->data->items + start * itemsize, slicelength * itemsize); return result; From 143178432b11343154f1a491cd4439f515779d50 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 11 Mar 2025 13:11:20 -0400 Subject: [PATCH 20/32] regen clinic --- Modules/arraymodule.c | 10 +++++----- Modules/clinic/arraymodule.c.h | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 67d3660b75cd01..4aab445bbffabc 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1252,7 +1252,7 @@ Return a copy of the array. static PyObject * array_array___deepcopy___impl(arrayobject *self, PyObject *unused) -/*[clinic end generated code: output=703b4c412feaaf31 input=2405ecb4933748c4]*/ +/*[clinic end generated code: output=703b4c412feaaf31 input=1a29f718f5b8a1dc]*/ { return array_array___copy___impl(self); } @@ -1528,7 +1528,7 @@ Return number of occurrences of v in the array. static PyObject * array_array_count_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=93ead26a2affb739 input=d9bce9d65e39d1f5]*/ +/*[clinic end generated code: output=93ead26a2affb739 input=c12c0042c1d0e27e]*/ { Py_ssize_t count = 0; Py_ssize_t i; @@ -1640,7 +1640,7 @@ Remove the first occurrence of v in the array. static PyObject * array_array_remove_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=f2a24e288ecb2a35 input=0b1e5aed25590027]*/ +/*[clinic end generated code: output=f2a24e288ecb2a35 input=78bef3fd40e62f7a]*/ { Py_ssize_t i; @@ -1794,7 +1794,7 @@ Append new value v to the end of the array. static PyObject * array_array_append_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=2f1e8cbad70c2a8b input=0b98d9d78e78f0fa]*/ +/*[clinic end generated code: output=2f1e8cbad70c2a8b input=9cdd897c66a40c3f]*/ { if (ins1(self, Py_SIZE(self), v) != 0) return NULL; @@ -2022,7 +2022,7 @@ Append items to array from list. static PyObject * array_array_fromlist_impl(arrayobject *self, PyObject *list) -/*[clinic end generated code: output=6c23733a68dd68df input=be2605a96c49680f]*/ +/*[clinic end generated code: output=6c23733a68dd68df input=c7c056aaf85d997a]*/ { Py_ssize_t n; diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 6d52ecdedf5098..569c116ca7266c 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -70,7 +70,7 @@ static PyObject * array_array___deepcopy___impl(arrayobject *self, PyObject *unused); static PyObject * -array_array___deepcopy__(arrayobject *self, PyObject *unused) +array_array___deepcopy__(PyObject *self, PyObject *unused) { PyObject *return_value = NULL; @@ -94,7 +94,7 @@ static PyObject * array_array_count_impl(arrayobject *self, PyObject *v); static PyObject * -array_array_count(arrayobject *self, PyObject *v) +array_array_count(PyObject *self, PyObject *v) { PyObject *return_value = NULL; @@ -166,7 +166,7 @@ static PyObject * array_array_remove_impl(arrayobject *self, PyObject *v); static PyObject * -array_array_remove(arrayobject *self, PyObject *v) +array_array_remove(PyObject *self, PyObject *v) { PyObject *return_value = NULL; @@ -351,7 +351,7 @@ static PyObject * array_array_append_impl(arrayobject *self, PyObject *v); static PyObject * -array_array_append(arrayobject *self, PyObject *v) +array_array_append(PyObject *self, PyObject *v) { PyObject *return_value = NULL; @@ -530,7 +530,7 @@ static PyObject * array_array_fromlist_impl(arrayobject *self, PyObject *list); static PyObject * -array_array_fromlist(arrayobject *self, PyObject *list) +array_array_fromlist(PyObject *self, PyObject *list) { PyObject *return_value = NULL; @@ -846,3 +846,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } +/*[clinic end generated code: output=46ca729af0fdb7cd input=a9049054013a1b77]*/ From 409239c9fe4cf43d0756163efc59e2257ac98e4f Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 15 Mar 2025 13:31:02 -0400 Subject: [PATCH 21/32] requested changes --- Lib/test/test_array.py | 50 ----------- Modules/arraymodule.c | 194 ++++++++++++++++++----------------------- 2 files changed, 87 insertions(+), 157 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 5ad36c799e92c6..75c4b10adbbe02 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1953,56 +1953,6 @@ def iter_reduce(b, a, it): # iterator stuff self.check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) - @unittest.skipUnless(support.check_sanitizer(thread=True), 'meant for tsan') - @threading_helper.reap_threads - @threading_helper.requires_working_threading() - def test_free_threading_tsan(self): - def copy_back_and_forth(b, a, count): - b.wait() - for _ in range(count): - a[0] = a[1] - a[1] = a[0] - - def append_and_pop(b, a, count): - v = a[0] - b.wait() - for _ in range(count): - a.append(v) - a.pop() - - def copy_random(b, a, count): - end = len(a) - 1 - idxs = [(random.randint(0, end), random.randint(0, end)) for _ in range(count)] - b.wait() - for src, dst in idxs: - a[dst] = a[src] - - # comment these back in if resize is made atomically quiet - # def extend_range(b, a, count): - # b.wait() - # for _ in range(count): - # a.extend(range(10)) - - # def extend_self(b, a, count): - # c = a[:] - # b.wait() - # for _ in range(count): - # a.extend(c) - - for tc in typecodes: - if tc in 'uw': - a = array.array(tc, 'ab') - else: - a = array.array(tc, [0, 1]) - - self.check([copy_back_and_forth] * 10, a, 100) - self.check([append_and_pop] * 10, a, 100) - # self.check([copy_back_and_forth] * 10 + [extend_self], a, 100) # comment this in if resize is made atomically quiet - - if tc not in 'uw': - # self.check([copy_back_and_forth] * 10 + [extend_range], a, 100) # comment this in if resize is made atomically quiet - self.check([copy_random] * 10, a * 5, 100) - if __name__ == "__main__": unittest.main() diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 4aab445bbffabc..c3921575e4765f 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -73,8 +73,8 @@ typedef struct { PyObject *str_iter; } array_state; -static inline -Py_ssize_t PyArray_GET_SIZE(PyObject *op) { +static inline Py_ssize_t +PyArray_GET_SIZE(PyObject *op) { arrayobject *ao = (arrayobject *)op; #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(ao)->ob_size)); @@ -149,16 +149,10 @@ enum machine_format_code { #define array_Check(op, state) PyObject_TypeCheck(op, state->ArrayType) -static inline char * -arraydata_safeitems(arraydata *data) -{ - return data == NULL ? NULL : data->items; -} - static arraydata * arraydata_alloc(Py_ssize_t size, int itemsize) { - if (size > PY_SSIZE_T_MAX/itemsize - 1) { + if ((size_t) size > (PY_SSIZE_T_MAX - sizeof(arraydata))/itemsize - 1) { return NULL; } arraydata *data = (arraydata *)PyMem_Malloc(sizeof(arraydata) + size * itemsize); @@ -202,7 +196,7 @@ arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int ite static arraydata * arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) { - if (size > PY_SSIZE_T_MAX/itemsize - 1) { + if ((size_t) size > (PY_SSIZE_T_MAX - sizeof(arraydata))/itemsize - 1) { return NULL; } data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); @@ -215,6 +209,12 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #endif +static inline char * +array_items_ptr(arrayobject *self) +{ + return self->data == NULL ? NULL : self->data->items; +} + static int array_resize(arrayobject *self, Py_ssize_t newsize) { @@ -323,8 +323,8 @@ in bounds; that's the responsibility of the caller. static PyObject * b_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) (signed char) FT_ATOMIC_LOAD_CHAR_RELAXED(items[i])); + long x = ((signed char *)items)[i]; + return PyLong_FromLong(x); } static int @@ -348,7 +348,7 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_CHAR_RELAXED(items[i], (char) x); + ((char *)items)[i] = (char)x; } return 0; } @@ -356,8 +356,8 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * BB_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) FT_ATOMIC_LOAD_UCHAR_RELAXED(((unsigned char *) items)[i])); + long x = ((unsigned char *)items)[i]; + return PyLong_FromLong(x); } static int @@ -369,7 +369,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_UCHAR_RELAXED(((unsigned char *) items)[i], x); + ((unsigned char *)items)[i] = x; } return 0; } @@ -377,13 +377,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(char *items, Py_ssize_t i) { -#if SIZEOF_WCHAR_T > 2 - return PyUnicode_FromOrdinal( - (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); -#else - return PyUnicode_FromOrdinal( - (wchar_t) FT_ATOMIC_LOAD_SHORT_RELAXED(((wchar_t *) items)[i])); -#endif + return PyUnicode_FromOrdinal(((wchar_t *) items)[i]); } static int @@ -418,11 +412,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { -#if SIZEOF_WCHAR_T > 2 - FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); -#else - FT_ATOMIC_STORE_SHORT_RELAXED(((wchar_t *) items)[i], w); -#endif + ((wchar_t *)items)[i] = w; } return 0; } @@ -430,8 +420,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * w_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal( - (Py_UCS4) FT_ATOMIC_LOAD_UINT32_RELAXED(((Py_UCS4 *) items)[i])); + return PyUnicode_FromOrdinal(((Py_UCS4 *) items)[i]); } static int @@ -453,8 +442,7 @@ w_setitem(char *items, Py_ssize_t i, PyObject *v) } if (i >= 0) { - Py_UCS4 w = PyUnicode_READ_CHAR(v, 0); - FT_ATOMIC_STORE_UINT32_RELAXED(((Py_UCS4 *) items)[i], w); + ((Py_UCS4 *)items)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } @@ -462,8 +450,7 @@ w_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * h_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) FT_ATOMIC_LOAD_SHORT_RELAXED(((short *) items)[i])); + return PyLong_FromLong((long) ((short *)items)[i]); } @@ -476,7 +463,7 @@ h_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_SHORT_RELAXED(((short *) items)[i], x); + ((short *)items)[i] = x; } return 0; } @@ -484,8 +471,7 @@ h_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * HH_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) FT_ATOMIC_LOAD_USHORT_RELAXED(((unsigned short *) items)[i])); + return PyLong_FromLong((long) ((unsigned short *)items)[i]); } static int @@ -508,7 +494,7 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_USHORT_RELAXED(((unsigned short *) items)[i], (unsigned short) x); + ((short *)items)[i] = (short)x; } return 0; } @@ -516,8 +502,7 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * i_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) FT_ATOMIC_LOAD_INT_RELAXED(((int *) items)[i])); + return PyLong_FromLong((long) ((int *)items)[i]); } static int @@ -529,7 +514,7 @@ i_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_INT_RELAXED(((int *)items)[i], x); + ((int *)items)[i] = x; } return 0; } @@ -538,7 +523,7 @@ static PyObject * II_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) FT_ATOMIC_LOAD_UINT_RELAXED(((unsigned int *) items)[i])); + (unsigned long) ((unsigned int *)items)[i]); } static int @@ -570,7 +555,7 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_UINT_RELAXED(((unsigned int *) items)[i], (unsigned int) x); + ((unsigned int *)items)[i] = (unsigned int)x; } if (do_decref) { @@ -582,8 +567,7 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * l_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - FT_ATOMIC_LOAD_LONG_RELAXED(((long *) items)[i])); + return PyLong_FromLong(((long *)items)[i]); } static int @@ -594,7 +578,7 @@ l_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_LONG_RELAXED(((long *) items)[i], x); + ((long *)items)[i] = x; } return 0; } @@ -602,8 +586,7 @@ l_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * LL_getitem(char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLong( - FT_ATOMIC_LOAD_ULONG_RELAXED(((unsigned long *) items)[i])); + return PyLong_FromUnsignedLong(((unsigned long *)items)[i]); } static int @@ -627,7 +610,7 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_ULONG_RELAXED(((unsigned long *) items)[i], x); + ((unsigned long *)items)[i] = x; } if (do_decref) { @@ -639,8 +622,7 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * q_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLongLong( - FT_ATOMIC_LOAD_LLONG_RELAXED(((long long *) items)[i])); + return PyLong_FromLongLong(((long long *)items)[i]); } static int @@ -651,7 +633,7 @@ q_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_LLONG_RELAXED(((long long *) items)[i], x); + ((long long *)items)[i] = x; } return 0; } @@ -660,7 +642,7 @@ static PyObject * QQ_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - FT_ATOMIC_LOAD_ULLONG_RELAXED(((unsigned long long *) items)[i])); + ((unsigned long long *)items)[i]); } static int @@ -684,7 +666,7 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_ULLONG_RELAXED(((unsigned long long *) items)[i], x); + ((unsigned long long *)items)[i] = x; } if (do_decref) { @@ -696,8 +678,7 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * f_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble( - (double) FT_ATOMIC_LOAD_FLOAT_RELAXED(((float *) items)[i])); + return PyFloat_FromDouble((double) ((float *)items)[i]); } static int @@ -708,7 +689,7 @@ f_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_FLOAT_RELAXED(((float *) items)[i], x); + ((float *)items)[i] = x; } return 0; } @@ -716,8 +697,7 @@ f_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * d_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble( - FT_ATOMIC_LOAD_DOUBLE_RELAXED(((double *) items)[i])); + return PyFloat_FromDouble(((double *)items)[i]); } static int @@ -728,7 +708,7 @@ d_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_DOUBLE_RELAXED(((double *) items)[i], x); + ((double *)items)[i] = x; } return 0; } @@ -994,7 +974,7 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) if (array_resize(self, n+1) == -1) return -1; - items = self->data->items; + items = array_items_ptr(self); if (where < 0) { where += n; if (where < 0) @@ -1072,7 +1052,7 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Fast path: arrays with same types can have their buffers compared directly */ Py_ssize_t common_length = Py_MIN(Py_SIZE(va), Py_SIZE(wa)); - int result = va->ob_descr->compareitems(va->data->items, wa->data->items, + int result = va->ob_descr->compareitems(array_items_ptr(va), array_items_ptr(wa), common_length); if (result == 0) goto compare_sizes; @@ -1095,11 +1075,11 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(va, i, va->data->items); + vi = getarrayitem(va, i, array_items_ptr(va)); if (vi == NULL) { return NULL; } - wi = getarrayitem(wa, i, wa->data->items); + wi = getarrayitem(wa, i, array_items_ptr(wa)); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -1203,7 +1183,7 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - memcpy(np->data->items, a->data->items + ilow * a->ob_descr->itemsize, + memcpy(array_items_ptr(np), array_items_ptr(a) + ilow * a->ob_descr->itemsize, (ihigh-ilow) * a->ob_descr->itemsize); } return (PyObject *)np; @@ -1286,11 +1266,11 @@ array_concat_lock_held(PyObject *op, PyObject *bb) return NULL; } if (Py_SIZE(a) > 0) { - memcpy(np->data->items, a->data->items, Py_SIZE(a)*a->ob_descr->itemsize); + memcpy(array_items_ptr(np), array_items_ptr(a), Py_SIZE(a)*a->ob_descr->itemsize); } if (Py_SIZE(b) > 0) { - memcpy(np->data->items + Py_SIZE(a)*a->ob_descr->itemsize, - b->data->items, Py_SIZE(b)*b->ob_descr->itemsize); + memcpy(array_items_ptr(np) + Py_SIZE(a)*a->ob_descr->itemsize, + array_items_ptr(b), Py_SIZE(b)*b->ob_descr->itemsize); } return (PyObject *)np; #undef b @@ -1328,7 +1308,7 @@ array_repeat_lock_held(PyObject *op, Py_ssize_t n) const Py_ssize_t oldbytes = array_length * a->ob_descr->itemsize; const Py_ssize_t newbytes = oldbytes * n; - _PyBytes_Repeat(np->data->items, newbytes, a->data->items, oldbytes); + _PyBytes_Repeat(array_items_ptr(np), newbytes, array_items_ptr(a), oldbytes); return (PyObject *)np; } @@ -1359,7 +1339,7 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) ihigh = ilow; else if (ihigh > Py_SIZE(a)) ihigh = Py_SIZE(a); - item = a->data->items; + item = array_items_ptr(a); d = ihigh-ilow; /* Issue #4509: If the array has exported buffers and the slice assignment would change the size of the array, fail early to make @@ -1444,8 +1424,8 @@ array_do_extend_lock_held(array_state *state, arrayobject *self, PyObject *bb) if (array_resize(self, size) == -1) return -1; if (bbsize > 0) { - memcpy(self->data->items + oldsize * self->ob_descr->itemsize, - b->data->items, bbsize * b->ob_descr->itemsize); + memcpy(array_items_ptr(self) + oldsize * self->ob_descr->itemsize, + array_items_ptr(b), bbsize * b->ob_descr->itemsize); } return 0; @@ -1500,7 +1480,7 @@ array_inplace_repeat_lock_held(PyObject *op, Py_ssize_t n) if (array_resize(self, n * array_size) == -1) return NULL; - _PyBytes_Repeat(self->data->items, n*size, self->data->items, size); + _PyBytes_Repeat(array_items_ptr(self), n*size, array_items_ptr(self), size); } return Py_NewRef(self); } @@ -1537,7 +1517,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, self->data->items); + selfi = getarrayitem(self, i, array_items_ptr(self)); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1585,7 +1565,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, self->data->items); + selfi = getarrayitem(self, i, array_items_ptr(self)); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1648,7 +1628,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, self->data->items); + selfi = getarrayitem(self, i, array_items_ptr(self)); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1694,7 +1674,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem(self, i, self->data->items); + v = getarrayitem(self, i, array_items_ptr(self)); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -1765,7 +1745,7 @@ array_array_buffer_info_impl(arrayobject *self) if (!retval) return NULL; - v = PyLong_FromVoidPtr(arraydata_safeitems(self->data)); + v = PyLong_FromVoidPtr(array_items_ptr(self)); if (v == NULL) { Py_DECREF(retval); return NULL; @@ -1822,14 +1802,14 @@ array_array_byteswap_impl(arrayobject *self) case 1: break; case 2: - for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 2) { + for (p = array_items_ptr(self), i = Py_SIZE(self); --i >= 0; p += 2) { char p0 = p[0]; p[0] = p[1]; p[1] = p0; } break; case 4: - for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 4) { + for (p = array_items_ptr(self), i = Py_SIZE(self); --i >= 0; p += 4) { char p0 = p[0]; char p1 = p[1]; p[0] = p[3]; @@ -1839,7 +1819,7 @@ array_array_byteswap_impl(arrayobject *self) } break; case 8: - for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 8) { + for (p = array_items_ptr(self), i = Py_SIZE(self); --i >= 0; p += 8) { char p0 = p[0]; char p1 = p[1]; char p2 = p[2]; @@ -1880,8 +1860,8 @@ array_array_reverse_impl(arrayobject *self) assert((size_t)itemsize <= sizeof(tmp)); if (Py_SIZE(self) > 1) { - for (p = self->data->items, - q = self->data->items + (Py_SIZE(self) - 1)*itemsize; + for (p = array_items_ptr(self), + q = array_items_ptr(self) + (Py_SIZE(self) - 1)*itemsize; p < q; p += itemsize, q -= itemsize) { /* memory areas guaranteed disjoint, so memcpy @@ -1990,7 +1970,7 @@ array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) assert(state != NULL); for (i = 0; i < nblocks; i++) { - char *ptr = self->data->items + i*BLOCKSIZE; + char *ptr = array_items_ptr(self) + i*BLOCKSIZE; Py_ssize_t size = BLOCKSIZE; PyObject *bytes, *res; @@ -2038,7 +2018,7 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) return NULL; for (i = 0; i < n; i++) { PyObject *v = PyList_GET_ITEM(list, i); - if (setarrayitem(self, Py_SIZE(self) - n + i, v, self->data->items) != 0) { + if (setarrayitem(self, Py_SIZE(self) - n + i, v, array_items_ptr(self)) != 0) { array_resize(self, old_size); return NULL; } @@ -2070,7 +2050,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem(self, i, self->data->items); + PyObject *v = getarrayitem(self, i, array_items_ptr(self)); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -2119,7 +2099,7 @@ array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) if (array_resize(self, old_size + n) == -1) { return NULL; } - memcpy(self->data->items + old_size * itemsize, + memcpy(array_items_ptr(self) + old_size * itemsize, buffer->buf, n * itemsize); } Py_RETURN_NONE; @@ -2137,7 +2117,7 @@ array_array_tobytes_impl(arrayobject *self) /*[clinic end generated code: output=87318e4edcdc2bb6 input=c4d44d5499d2320f]*/ { if (Py_SIZE(self) <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { - return PyBytes_FromStringAndSize(arraydata_safeitems(self->data), + return PyBytes_FromStringAndSize(array_items_ptr(self), Py_SIZE(self) * self->ob_descr->itemsize); } else { return PyErr_NoMemory(); @@ -2182,7 +2162,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) // must not fail PyUnicode_AsWideChar( - ustr, ((wchar_t *)arraydata_safeitems(self->data)) + old_size, ustr_length); + ustr, ((wchar_t *)array_items_ptr(self)) + old_size, ustr_length); } } else { // typecode == 'w' @@ -2198,7 +2178,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) } // must not fail - Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)arraydata_safeitems(self->data)) + old_size, + Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)array_items_ptr(self)) + old_size, ustr_length, 0); assert(u != NULL); (void)u; // Suppress unused_variable warning. @@ -2229,11 +2209,11 @@ array_array_tounicode_impl(arrayobject *self) return NULL; } if (typecode == 'u') { - return PyUnicode_FromWideChar((wchar_t *) arraydata_safeitems(self->data), Py_SIZE(self)); + return PyUnicode_FromWideChar((wchar_t *) array_items_ptr(self), Py_SIZE(self)); } else { // typecode == 'w' int byteorder = 0; // native byteorder - return PyUnicode_DecodeUTF32((const char *) arraydata_safeitems(self->data), Py_SIZE(self) * 4, + return PyUnicode_DecodeUTF32((const char *) array_items_ptr(self), Py_SIZE(self) * 4, NULL, &byteorder); } } @@ -2856,7 +2836,7 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) if (result == NULL) return NULL; memcpy(((arrayobject *)result)->data->items, - self->data->items + start * itemsize, + array_items_ptr(self) + start * itemsize, slicelength * itemsize); return result; } @@ -2869,7 +2849,7 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) for (cur = start, i = 0; i < slicelength; cur += step, i++) { memcpy(ar->data->items + i*itemsize, - self->data->items + cur*itemsize, + array_items_ptr(self) + cur*itemsize, itemsize); } @@ -2940,7 +2920,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) slicelength = 1; } else - return setarrayitem(self, i, value, self->data->items); + return setarrayitem(self, i, value, array_items_ptr(self)); } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -2999,8 +2979,8 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (step == 1) { if (slicelength > needed) { - memmove(self->data->items + (start + needed) * itemsize, - self->data->items + stop * itemsize, + memmove(array_items_ptr(self) + (start + needed) * itemsize, + array_items_ptr(self) + stop * itemsize, (Py_SIZE(self) - stop) * itemsize); if (array_resize(self, Py_SIZE(self) + needed - slicelength) < 0) @@ -3010,12 +2990,12 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (array_resize(self, Py_SIZE(self) + needed - slicelength) < 0) return -1; - memmove(self->data->items + (start + needed) * itemsize, - self->data->items + stop * itemsize, + memmove(array_items_ptr(self) + (start + needed) * itemsize, + array_items_ptr(self) + stop * itemsize, (Py_SIZE(self) - start - needed) * itemsize); } if (needed > 0) - memcpy(self->data->items + start * itemsize, + memcpy(array_items_ptr(self) + start * itemsize, other->data->items, needed * itemsize); return 0; } @@ -3035,14 +3015,14 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (cur + step >= (size_t)Py_SIZE(self)) lim = Py_SIZE(self) - cur - 1; - memmove(self->data->items + (cur - i) * itemsize, - self->data->items + (cur + 1) * itemsize, + memmove(array_items_ptr(self) + (cur - i) * itemsize, + array_items_ptr(self) + (cur + 1) * itemsize, lim * itemsize); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->data->items + (cur-slicelength) * itemsize, - self->data->items + cur * itemsize, + memmove(array_items_ptr(self) + (cur-slicelength) * itemsize, + array_items_ptr(self) + cur * itemsize, (Py_SIZE(self) - cur) * itemsize); } if (array_resize(self, Py_SIZE(self) - slicelength) < 0) @@ -3062,7 +3042,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) } for (cur = start, i = 0; i < slicelength; cur += step, i++) { - memcpy(self->data->items + cur * itemsize, + memcpy(array_items_ptr(self) + cur * itemsize, other->data->items + i * itemsize, itemsize); } @@ -3113,7 +3093,7 @@ array_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) return -1; } - view->buf = (void *)arraydata_safeitems(self->data); + view->buf = (void *)array_items_ptr(self); view->obj = Py_NewRef(self); if (view->buf == NULL) view->buf = (void *)emptybuf; @@ -3285,7 +3265,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) else if (initial != NULL && array_Check(initial, state) && len > 0) { arrayobject *self = (arrayobject *)a; arrayobject *other = (arrayobject *)initial; - memcpy(self->data->items, other->data->items, len * other->ob_descr->itemsize); + memcpy(array_items_ptr(self), other->data->items, len * other->ob_descr->itemsize); } if (it != NULL) { if (array_iter_extend((arrayobject *)a, it) == -1) { From a6f17c90fc4b1d6a51192e30678a76cdc03d61e2 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 17 Mar 2025 18:15:27 -0400 Subject: [PATCH 22/32] more requested changes --- Modules/arraymodule.c | 116 ++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 67 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index c3921575e4765f..f33f6d4fea186c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -44,7 +44,7 @@ struct arraydescr { typedef struct { Py_ssize_t allocated; - char items[]; + _Alignas(8) char items[]; } arraydata; typedef struct arrayobject { @@ -73,7 +73,7 @@ typedef struct { PyObject *str_iter; } array_state; -static inline Py_ssize_t +static Py_ssize_t PyArray_GET_SIZE(PyObject *op) { arrayobject *ao = (arrayobject *)op; #ifdef Py_GIL_DISABLED @@ -152,9 +152,6 @@ enum machine_format_code { static arraydata * arraydata_alloc(Py_ssize_t size, int itemsize) { - if ((size_t) size > (PY_SSIZE_T_MAX - sizeof(arraydata))/itemsize - 1) { - return NULL; - } arraydata *data = (arraydata *)PyMem_Malloc(sizeof(arraydata) + size * itemsize); if (data == NULL) { return NULL; @@ -180,25 +177,11 @@ arraydata_free(arraydata *data, bool use_qsbr) #endif } -static arraydata * -arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int itemsize, bool use_qsbr) -{ - arraydata_free(data, use_qsbr); - data = arraydata_alloc(newsize, itemsize); - if (data != NULL) { - memcpy(data->items, newitems, newsize * itemsize); - } - return data; -} - #ifndef Py_GIL_DISABLED static arraydata * arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) { - if ((size_t) size > (PY_SSIZE_T_MAX - sizeof(arraydata))/itemsize - 1) { - return NULL; - } data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); if (data == NULL) { return NULL; @@ -209,7 +192,7 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #endif -static inline char * +static char * array_items_ptr(arrayobject *self) { return self->data == NULL ? NULL : self->data->items; @@ -805,25 +788,25 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return (PyObject *) op; } -static inline int +static int valid_index(Py_ssize_t i, Py_ssize_t limit) { return (size_t) i < (size_t) limit; } -static inline PyObject * -getarrayitem(arrayobject *ap, Py_ssize_t i, char *items) +static PyObject * +getarrayitem(arrayobject *ap, Py_ssize_t i, arraydata *data) { #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(ap)); assert(array_Check(ap, state)); #ifdef Py_GIL_DISABLED - assert(valid_index(i, (_Py_CONTAINER_OF(items, arraydata, items))->allocated)); + assert(valid_index(i, data->allocated)); #else assert(valid_index(i, Py_SIZE(ap))); #endif #endif - return (*ap->ob_descr->getitem)(items, i); + return (*ap->ob_descr->getitem)(data->items, i); } static PyObject * @@ -841,7 +824,7 @@ getarrayitem_locked(PyObject *op, Py_ssize_t i) } else { arrayobject *ap = (arrayobject *)op; - ret = getarrayitem(ap, i, ap->data->items); + ret = getarrayitem(ap, i, ap->data); } Py_END_CRITICAL_SECTION(); return ret; @@ -850,7 +833,7 @@ getarrayitem_locked(PyObject *op, Py_ssize_t i) #ifdef Py_GIL_DISABLED static PyObject * -getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) +getarrayitem_threadsafe(PyObject *op, Py_ssize_t i) { if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { return getarrayitem_locked(op, i); @@ -867,37 +850,34 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) if (!valid_index(i, data->allocated)) { return NULL; } - return getarrayitem(ap, i, data->items); - /* Could check size again here to make sure it hasn't changed but since - there isn't a well defined order between unsynchronized thread - operations there's no point. */ + return getarrayitem(ap, i, data); } #else // Py_GIL_DISABLED static PyObject * -getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) +getarrayitem_threadsafe(PyObject *op, Py_ssize_t i) { return getarrayitem_locked(op, i); } #endif // Py_GIL_DISABLED -static inline int -setarrayitem(arrayobject *ap, Py_ssize_t i, PyObject *v, char *items) +static int +setarrayitem(arrayobject *ap, Py_ssize_t i, PyObject *v, arraydata *data) { #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(ap)); assert(array_Check(ap, state)); - if (items != NULL) { + if (data != NULL) { #ifdef Py_GIL_DISABLED - assert(valid_index(i, (_Py_CONTAINER_OF(items, arraydata, items))->allocated)); + assert(valid_index(i, data->allocated)); #else assert(valid_index(i, Py_SIZE(ap))); #endif } #endif - return (*ap->ob_descr->setitem)(items, i, v); + return (*ap->ob_descr->setitem)(data->items, i, v); } static int @@ -916,7 +896,7 @@ setarrayitem_locked(PyObject *op, Py_ssize_t i, PyObject *v) } else { arrayobject *ap = arrayobject_CAST(op); - ret = setarrayitem(ap, i, v, ap->data->items); + ret = setarrayitem(ap, i, v, ap->data); } Py_END_CRITICAL_SECTION(); return ret; @@ -925,7 +905,7 @@ setarrayitem_locked(PyObject *op, Py_ssize_t i, PyObject *v) #ifdef Py_GIL_DISABLED static int -setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) +setarrayitem_threadsafe(PyObject *op, Py_ssize_t i, PyObject *v) { if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { return setarrayitem_locked(op, i, v); @@ -942,7 +922,7 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) if (!valid_index(i, data->allocated)) { goto error; } - return setarrayitem(ap, i, v, data->items); + return setarrayitem(ap, i, v, data); error: PyErr_SetString(PyExc_IndexError, "array index out of range"); @@ -952,7 +932,7 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) #else // Py_GIL_DISABLED static int -setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) +setarrayitem_threadsafe(PyObject *op, Py_ssize_t i, PyObject *v) { return setarrayitem_locked(op, i, v); } @@ -963,7 +943,6 @@ static int ins1(arrayobject *self, Py_ssize_t where, PyObject *v) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - char *items; Py_ssize_t n = Py_SIZE(self); if (v == NULL) { PyErr_BadInternalCall(); @@ -974,7 +953,6 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) if (array_resize(self, n+1) == -1) return -1; - items = array_items_ptr(self); if (where < 0) { where += n; if (where < 0) @@ -984,10 +962,10 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) where = n; /* appends don't need to call memmove() */ if (where != n) - memmove(items + (where+1)*self->ob_descr->itemsize, - items + where*self->ob_descr->itemsize, + memmove(array_items_ptr(self) + (where+1)*self->ob_descr->itemsize, + array_items_ptr(self) + where*self->ob_descr->itemsize, (n-where)*self->ob_descr->itemsize); - return setarrayitem(self, where, v, items); + return setarrayitem(self, where, v, self->data); } /* Methods */ @@ -1075,11 +1053,11 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(va, i, array_items_ptr(va)); + vi = getarrayitem(va, i, va->data); if (vi == NULL) { return NULL; } - wi = getarrayitem(wa, i, array_items_ptr(wa)); + wi = getarrayitem(wa, i, wa->data); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -1154,7 +1132,7 @@ array_length(PyObject *op) static PyObject * array_item(PyObject *op, Py_ssize_t i) { - PyObject *item = getarrayitem_maybe_locked(op, i); + PyObject *item = getarrayitem_threadsafe(op, i); if (item == NULL) { PyErr_SetString(PyExc_IndexError, "array index out of range"); } @@ -1363,7 +1341,7 @@ static int array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) { if (v != NULL) { - return setarrayitem_maybe_locked(op, i, v); + return setarrayitem_threadsafe(op, i, v); } int ret; Py_BEGIN_CRITICAL_SECTION(op); @@ -1517,7 +1495,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, array_items_ptr(self)); + selfi = getarrayitem(self, i, self->data); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1565,7 +1543,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, array_items_ptr(self)); + selfi = getarrayitem(self, i, self->data); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1589,7 +1567,7 @@ array_contains_lock_held(PyObject *op, PyObject *v) for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { arrayobject *ap = (arrayobject *)op; - PyObject *opi = getarrayitem(ap, i, ap->data->items); + PyObject *opi = getarrayitem(ap, i, ap->data); if (opi == NULL) return -1; cmp = PyObject_RichCompareBool(opi, v, Py_EQ); @@ -1628,7 +1606,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, array_items_ptr(self)); + selfi = getarrayitem(self, i, self->data); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1674,7 +1652,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem(self, i, array_items_ptr(self)); + v = getarrayitem(self, i, self->data); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -2018,7 +1996,7 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) return NULL; for (i = 0; i < n; i++) { PyObject *v = PyList_GET_ITEM(list, i); - if (setarrayitem(self, Py_SIZE(self) - n + i, v, array_items_ptr(self)) != 0) { + if (setarrayitem(self, Py_SIZE(self) - n + i, v, self->data) != 0) { array_resize(self, old_size); return NULL; } @@ -2050,7 +2028,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem(self, i, array_items_ptr(self)); + PyObject *v = getarrayitem(self, i, self->data); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -2920,7 +2898,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) slicelength = 1; } else - return setarrayitem(self, i, value, array_items_ptr(self)); + return setarrayitem(self, i, value, self->data); } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -3069,7 +3047,7 @@ array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) return -1; if (i < 0) i += PyArray_GET_SIZE(op); - return setarrayitem_maybe_locked(op, i, value); + return setarrayitem_threadsafe(op, i, value); } } @@ -3202,7 +3180,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) Py_DECREF(a); return NULL; } - if (setarrayitem(ap, i, v, ap->data->items) != 0) { + if (setarrayitem(ap, i, v, ap->data) != 0) { Py_DECREF(v); Py_DECREF(a); return NULL; @@ -3231,14 +3209,16 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (n > 0) { arrayobject *self = (arrayobject *)a; - self->data = arraydata_set_items(self->data, - (char *)ustr, n, sizeof(wchar_t), false); - PyMem_Free(ustr); + assert(self->data == NULL); + self->data = arraydata_alloc(n, sizeof(wchar_t)); if (self->data == NULL) { + PyMem_Free(ustr); Py_DECREF(a); PyErr_NoMemory(); return NULL; } + memcpy(self->data->items, (char *)ustr, n * sizeof(wchar_t)); + PyMem_Free(ustr); Py_SET_SIZE(self, n); } } @@ -3251,14 +3231,16 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) } arrayobject *self = (arrayobject *)a; - self->data = arraydata_set_items(self->data, - (char *)ustr, n, sizeof(Py_UCS4), false); - PyMem_Free(ustr); + assert(self->data == NULL); + self->data = arraydata_alloc(n, sizeof(Py_UCS4)); if (self->data == NULL) { + PyMem_Free(ustr); Py_DECREF(a); PyErr_NoMemory(); return NULL; } + memcpy(self->data->items, (char *)ustr, n * sizeof(Py_UCS4)); + PyMem_Free(ustr); Py_SET_SIZE(self, n); } } @@ -3508,7 +3490,7 @@ arrayiter_next(PyObject *op) assert(PyObject_TypeCheck(it, state->ArrayIterType)); assert(array_Check(ao, state)); #endif - PyObject *ret = getarrayitem_maybe_locked((PyObject *)ao, index); + PyObject *ret = getarrayitem_threadsafe((PyObject *)ao, index); if (ret != NULL) { FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, index + 1); } From b5d219ebe2d8305701b5a0be379411c0e9b38628 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 17 Mar 2025 18:30:52 -0400 Subject: [PATCH 23/32] __declspec(align(8)) for Windows --- Modules/arraymodule.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index f33f6d4fea186c..85d50f3ab8cea1 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,11 +42,22 @@ struct arraydescr { int is_signed; }; +#ifdef _MSC_VER + +typedef __declspec(align(8)) struct { + Py_ssize_t allocated; + char items[]; +} arraydata; + +#else + typedef struct { Py_ssize_t allocated; _Alignas(8) char items[]; } arraydata; +#endif + typedef struct arrayobject { PyObject_VAR_HEAD arraydata *data; From 2c82071a7106d3b9ca9e6695edbc7a1c35db3694 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 17 Mar 2025 19:00:31 -0400 Subject: [PATCH 24/32] shut up check-c-globals --- Modules/arraymodule.c | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 85d50f3ab8cea1..0b9419be27f993 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,22 +42,14 @@ struct arraydescr { int is_signed; }; -#ifdef _MSC_VER - -typedef __declspec(align(8)) struct { - Py_ssize_t allocated; - char items[]; -} arraydata; - -#else - typedef struct { - Py_ssize_t allocated; - _Alignas(8) char items[]; + union { + Py_ssize_t allocated; + double __alignment_padding; /* In case Py_ssize_t is 4 bytes */ + }; + char items[]; } arraydata; -#endif - typedef struct arrayobject { PyObject_VAR_HEAD arraydata *data; From 1ba50e97343f4f9765c643b64d49c8322371718c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 20 Mar 2025 07:17:49 -0400 Subject: [PATCH 25/32] alignment changes --- Modules/arraymodule.c | 13 ++++++++----- Tools/c-analyzer/c_parser/parser/_regexes.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 0b9419be27f993..dd018db779035c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,12 +42,15 @@ struct arraydescr { int is_signed; }; +#ifdef MS_WINDOWS +#define Py_ALIGN_AS(x) __declspec(align(x)) +#else +#define Py_ALIGN_AS(x) _Alignas(x) +#endif + typedef struct { - union { - Py_ssize_t allocated; - double __alignment_padding; /* In case Py_ssize_t is 4 bytes */ - }; - char items[]; + Py_ssize_t allocated; + Py_ALIGN_AS(8) char items[]; } arraydata; typedef struct arrayobject { diff --git a/Tools/c-analyzer/c_parser/parser/_regexes.py b/Tools/c-analyzer/c_parser/parser/_regexes.py index c1a8ab3ad2f15d..76f0f71a2f7724 100644 --- a/Tools/c-analyzer/c_parser/parser/_regexes.py +++ b/Tools/c-analyzer/c_parser/parser/_regexes.py @@ -59,6 +59,8 @@ def _ind(text, level=1, edges='both'): register | static | _Thread_local | + _Alignas | + _Alignof | typedef | const | @@ -154,6 +156,17 @@ def _ind(text, level=1, edges='both'): TYPE_QUALIFIER = r'(?: \b (?: const | volatile ) \b )' PTR_QUALIFIER = rf'(?: [*] (?: \s* {TYPE_QUALIFIER} )? )' +ALIGNMENT_SPECIFIER = textwrap.dedent(r''' + # alignment specifier + (?: + _Alignas + \s* [(] + [^)]* + [)] + ) + # end alignment specifier + ''') + TYPE_SPEC = textwrap.dedent(rf''' # type spec (?: @@ -318,6 +331,9 @@ def _ind(text, level=1, edges='both'): (?: # typed member (?: + (?: # + \s* {ALIGNMENT_SPECIFIER} \s* + )? # Technically it doesn't have to have a type... (?: # (?: {TYPE_QUALIFIER} \s* )? From 576aebf4ba5ed9dbae32c66d7df3305f1df950e8 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 20 Mar 2025 13:19:42 -0400 Subject: [PATCH 26/32] MS_WINDOWS -> _MSC_VER --- Modules/arraymodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index dd018db779035c..7155bebafb0c9f 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,7 +42,7 @@ struct arraydescr { int is_signed; }; -#ifdef MS_WINDOWS +#ifdef _MSC_VER #define Py_ALIGN_AS(x) __declspec(align(x)) #else #define Py_ALIGN_AS(x) _Alignas(x) From d4e53139d7b931bb171d881bf7b529d8521ea069 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 20 Mar 2025 13:38:16 -0400 Subject: [PATCH 27/32] #include "pycore_gc.h", something moved --- Modules/arraymodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 7155bebafb0c9f..5ade73348424fd 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -11,6 +11,7 @@ #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_gc.h" // _PyObject_GC_IS_SHARED() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" From affae8e57009b25d7746a8b2cf0ae3bf8b4ec7db Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 25 Mar 2025 10:54:04 -0400 Subject: [PATCH 28/32] remove NULL check in arraydata_free() --- Modules/arraymodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 5ade73348424fd..0424c48e281230 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -172,9 +172,7 @@ arraydata_free(arraydata *data, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - if (data != NULL) { - _PyMem_FreeDelayed(data); - } + _PyMem_FreeDelayed(data); } else { PyMem_Free(data); From 2e7132e8fc0ab579c6d86fa77bbe818aa51dbb52 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 5 May 2025 07:15:04 -0400 Subject: [PATCH 29/32] change to use new _Py_ALIGN_AS() macro --- Include/pymacro.h | 50 +++++++++++++++++++++---------------------- Modules/arraymodule.c | 9 ++------ 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h index 218987a80b0d91..f0ca334e87545c 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -25,8 +25,8 @@ // _Py_ALIGN_AS: this compiler's spelling of `alignas` keyword, -// We currently use alignas for free-threaded builds only; additional compat -// checking would be great before we add it to the default build. +// additional compat checking would be great since we added it to the default +// build. // Standards/compiler support: // - `alignas` is a keyword in C23 and C++11. // - `_Alignas` is a keyword in C11 @@ -38,30 +38,28 @@ // unsupported platforms, we don't redefine _Py_ALIGN_AS if it's already // defined. Note that defining it wrong (including defining it to nothing) will // cause ABI incompatibilities. -#ifdef Py_GIL_DISABLED -# ifndef _Py_ALIGN_AS -# ifdef __cplusplus -# if __cplusplus >= 201103L -# define _Py_ALIGN_AS(V) alignas(V) -# elif defined(__GNUC__) || defined(__clang__) -# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) -# elif defined(_MSC_VER) -# define _Py_ALIGN_AS(V) __declspec(align(V)) -# else -# define _Py_ALIGN_AS(V) alignas(V) -# endif -# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L -# define _Py_ALIGN_AS(V) alignas(V) -# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -# define _Py_ALIGN_AS(V) _Alignas(V) -# elif (defined(__GNUC__) || defined(__clang__)) -# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) -# elif defined(_MSC_VER) -# define _Py_ALIGN_AS(V) __declspec(align(V)) -# else -# define _Py_ALIGN_AS(V) _Alignas(V) -# endif -# endif +#ifndef _Py_ALIGN_AS +# ifdef __cplusplus +# if __cplusplus >= 201103L +# define _Py_ALIGN_AS(V) alignas(V) +# elif defined(__GNUC__) || defined(__clang__) +# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) +# elif defined(_MSC_VER) +# define _Py_ALIGN_AS(V) __declspec(align(V)) +# else +# define _Py_ALIGN_AS(V) alignas(V) +# endif +# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +# define _Py_ALIGN_AS(V) alignas(V) +# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define _Py_ALIGN_AS(V) _Alignas(V) +# elif (defined(__GNUC__) || defined(__clang__)) +# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) +# elif defined(_MSC_VER) +# define _Py_ALIGN_AS(V) __declspec(align(V)) +# else +# define _Py_ALIGN_AS(V) _Alignas(V) +# endif #endif /* Minimum value between x and y */ diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3d9387855d293f..4d1c9ab28a5fe7 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -43,15 +43,10 @@ struct arraydescr { int is_signed; }; -#ifdef _MSC_VER -#define Py_ALIGN_AS(x) __declspec(align(x)) -#else -#define Py_ALIGN_AS(x) _Alignas(x) -#endif - typedef struct { Py_ssize_t allocated; - Py_ALIGN_AS(8) char items[]; + _Py_ALIGN_AS(8) + char items[]; } arraydata; typedef struct arrayobject { From 1a509813d640d9c610facb0172479899c931bc54 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 11 Jun 2025 09:32:04 -0400 Subject: [PATCH 30/32] requested changes --- Modules/arraymodule.c | 81 ++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 51 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 7687302458a6ab..a8cbb731e20fba 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -153,10 +153,14 @@ enum machine_format_code { static arraydata * arraydata_alloc(Py_ssize_t size, int itemsize) { - arraydata *data = (arraydata *)PyMem_Malloc(sizeof(arraydata) + size * itemsize); + size_t bufsize = sizeof(arraydata) + size * itemsize; + arraydata *data = (arraydata *)PyMem_Malloc(bufsize); if (data == NULL) { return NULL; } +#ifdef Py_DEBUG + memset(data, 0, bufsize); +#endif data->allocated = size; return data; } @@ -191,7 +195,7 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #endif -static char * +static inline char * array_items_ptr(arrayobject *self) { return self->data == NULL ? NULL : self->data->items; @@ -316,9 +320,8 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore must use the next size up that is signed ('h') and manually do the overflow checking */ - if (!PyArg_Parse(v, "h;array item must be integer", &x)) { + if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; - } else if (x < -128) { PyErr_SetString(PyExc_OverflowError, "signed char is less than minimum"); @@ -329,9 +332,8 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) "signed char is greater than maximum"); return -1; } - if (i >= 0) { + if (i >= 0) ((char *)items)[i] = (char)x; - } return 0; } @@ -347,12 +349,10 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ - if (!PyArg_Parse(v, "b;array item must be integer", &x)) { + if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((unsigned char *)items)[i] = x; - } return 0; } @@ -441,12 +441,10 @@ h_setitem(char *items, Py_ssize_t i, PyObject *v) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ - if (!PyArg_Parse(v, "h;array item must be integer", &x)) { + if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((short *)items)[i] = x; - } return 0; } @@ -462,9 +460,8 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore must use the next size up and manually do the overflow checking */ - if (!PyArg_Parse(v, "i;array item must be integer", &x)) { + if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; - } else if (x < 0) { PyErr_SetString(PyExc_OverflowError, "unsigned short is less than minimum"); @@ -475,9 +472,8 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) "unsigned short is greater than maximum"); return -1; } - if (i >= 0) { + if (i >= 0) ((short *)items)[i] = (short)x; - } return 0; } @@ -492,12 +488,10 @@ i_setitem(char *items, Py_ssize_t i, PyObject *v) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ - if (!PyArg_Parse(v, "i;array item must be integer", &x)) { + if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((int *)items)[i] = x; - } return 0; } @@ -536,9 +530,8 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) { + if (i >= 0) ((unsigned int *)items)[i] = (unsigned int)x; - } if (do_decref) { Py_DECREF(v); @@ -556,12 +549,10 @@ static int l_setitem(char *items, Py_ssize_t i, PyObject *v) { long x; - if (!PyArg_Parse(v, "l;array item must be integer", &x)) { + if (!PyArg_Parse(v, "l;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((long *)items)[i] = x; - } return 0; } @@ -591,9 +582,8 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) { + if (i >= 0) ((unsigned long *)items)[i] = x; - } if (do_decref) { Py_DECREF(v); @@ -611,12 +601,10 @@ static int q_setitem(char *items, Py_ssize_t i, PyObject *v) { long long x; - if (!PyArg_Parse(v, "L;array item must be integer", &x)) { + if (!PyArg_Parse(v, "L;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((long long *)items)[i] = x; - } return 0; } @@ -647,9 +635,8 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) { + if (i >= 0) ((unsigned long long *)items)[i] = x; - } if (do_decref) { Py_DECREF(v); @@ -667,12 +654,10 @@ static int f_setitem(char *items, Py_ssize_t i, PyObject *v) { float x; - if (!PyArg_Parse(v, "f;array item must be float", &x)) { + if (!PyArg_Parse(v, "f;array item must be float", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((float *)items)[i] = x; - } return 0; } @@ -686,12 +671,10 @@ static int d_setitem(char *items, Py_ssize_t i, PyObject *v) { double x; - if (!PyArg_Parse(v, "d;array item must be float", &x)) { + if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((double *)items)[i] = x; - } return 0; } @@ -700,11 +683,9 @@ d_setitem(char *items, Py_ssize_t i, PyObject *v) code##_compareitems(const void *lhs, const void *rhs, Py_ssize_t length) \ { \ const type *a = lhs, *b = rhs; \ - for (Py_ssize_t i = 0; i < length; ++i) { \ - if (a[i] != b[i]) { \ + for (Py_ssize_t i = 0; i < length; ++i) \ + if (a[i] != b[i]) \ return a[i] < b[i] ? -1 : 1; \ - } \ - } \ return 0; \ } @@ -984,9 +965,7 @@ array_dealloc(PyObject *op) arrayobject *self = arrayobject_CAST(op); if (self->ob_exports > 0) { - PyErr_SetString(PyExc_SystemError, - "deallocated array object has exported buffers"); - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("deallocated array object has exported buffers"); } if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(op); From f023e7718e9cf242c4e13c1ead0328c2cd81bf7c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 13 Jun 2025 12:41:09 -0400 Subject: [PATCH 31/32] shut up UBSan --- Modules/arraymodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a8cbb731e20fba..fcb3b24ab3319a 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -857,7 +857,7 @@ setarrayitem(arrayobject *ap, Py_ssize_t i, PyObject *v, arraydata *data) #endif } #endif - return (*ap->ob_descr->setitem)(data->items, i, v); + return (*ap->ob_descr->setitem)(data == NULL ? NULL : data->items, i, v); } static int From 6d50e9a64f76c6209e4676d8dbfd43a23196696e Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 6 Jul 2025 07:58:23 -0400 Subject: [PATCH 32/32] add size to _PyMem_FreeDelayed() --- Modules/arraymodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index b0a768b7db5afe..80f2d6aebaf04e 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -171,7 +171,7 @@ arraydata_free(arraydata *data, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(data); + _PyMem_FreeDelayed(data, data == NULL ? 0 : data->allocated); } else { PyMem_Free(data); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy