gh-98724: Fix the Py_CLEAR() macro in the limited C API

vstinner · vstinner · commit 5dac4378e77d · 2022-12-09T00:02:58.000+01:00
If _Py_TYPEOF() is not available, the limited C API now implements
the Py_CLEAR() macro as a function call which hides implementation
details. The function uses memcpy() of &lt;string.h&gt; which is not
included by &lt;Python.h&gt;.

* Add private _Py_Clear() function.
* Add _Py_Clear() to Misc/stable_abi.toml.
diff --git a/Include/object.h b/Include/object.h
@@ -612,7 +612,13 @@ static inline void Py_DECREF(PyObject *op)
  * and so avoid type punning. Otherwise, use memcpy() which causes type erasure
  * and so prevents the compiler to reuse an old cached 'op' value after
  * Py_CLEAR().
+ *
+ * If _Py_TYPEOF() is not available, the limited C API implementation uses a
+ * function call to hide implementation details. The function uses memcpy() of
+ * <string.h> which is not included by <Python.h>.
  */
+PyAPI_FUNC(void) _Py_Clear(PyObject **pobj);
+
 #ifdef _Py_TYPEOF
 #define Py_CLEAR(op) \
     do { \
@@ -623,6 +629,8 @@ static inline void Py_DECREF(PyObject *op)
             Py_DECREF(_tmp_old_op); \
         } \
     } while (0)
+#elif defined(Py_LIMITED_API)
+#define Py_CLEAR(op) _Py_Clear(_Py_CAST(PyObject**, &(op)))
 #else
 #define Py_CLEAR(op) \
     do { \
diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py
diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml
@@ -2386,3 +2386,7 @@
     added = '3.12'  # Before 3.12, available in "structmember.h" w/o Py_ prefix
 [const.Py_AUDIT_READ]
     added = '3.12'  # Before 3.12, available in "structmember.h"
+
+[function._Py_Clear]
+    added = '3.12'
+    abi_only = true
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
@@ -2593,27 +2593,53 @@ test_set_type_size(PyObject *self, PyObject *Py_UNUSED(ignored))
 static PyObject*
 test_py_clear(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
+    PyObject *list = PyList_New(0);
+    if (list == NULL) {
+        return NULL;
+    }
+    assert(Py_REFCNT(list) == 1);
+
     // simple case with a variable
-    PyObject *obj = PyList_New(0);
+    PyObject *obj = Py_NewRef(list);
     if (obj == NULL) {
-        return NULL;
+        goto error;
     }
+    assert(Py_REFCNT(list) == 2);
     Py_CLEAR(obj);
+    assert(Py_REFCNT(list) == 1);
     assert(obj == NULL);
 
+    // test _Py_Clear() used by the limited C API
+    PyObject *obj2 = Py_NewRef(list);
+    if (obj2 == NULL) {
+        goto error;
+    }
+    assert(Py_REFCNT(list) == 2);
+    _Py_Clear(&obj2);
+    assert(Py_REFCNT(list) == 1);
+    assert(obj2 == NULL);
+
     // gh-98724: complex case, Py_CLEAR() argument has a side effect
     PyObject* array[1];
-    array[0] = PyList_New(0);
+    array[0] = Py_NewRef(list);
     if (array[0] == NULL) {
-        return NULL;
+        goto error;
     }
 
     PyObject **p = array;
+    assert(Py_REFCNT(list) == 2);
     Py_CLEAR(*p++);
+    assert(Py_REFCNT(list) == 1);
     assert(array[0] == NULL);
     assert(p == array + 1);
 
+    assert(Py_REFCNT(list) == 1);
+    Py_DECREF(list);
     Py_RETURN_NONE;
+
+error:
+    Py_DECREF(list);
+    return NULL;
 }
 
 
diff --git a/Objects/object.c b/Objects/object.c
@@ -2446,6 +2446,23 @@ int Py_IsFalse(PyObject *x)
     return Py_Is(x, Py_False);
 }
 
+void _Py_Clear(PyObject **pobj)
+{
+    PyObject *old_obj = *pobj;
+    if (old_obj == NULL) {
+        return;
+    }
+
+    // gh-99701: In the limited C API, the Py_CLEAR(obj) macro has a type
+    // punning issue, it casts '&obj' to PyObject** to call _Py_Clear(). Use
+    // memcpy() instead of a simple assignment to cause type erasure and so
+    // prevent the compiler to reuse an old cached 'obj' value after
+    // Py_CLEAR().
+    PyObject *null_ptr = NULL;
+    memcpy(pobj, &null_ptr, sizeof(PyObject*));
+    Py_DECREF(old_obj);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/PC/python3dll.c b/PC/python3dll.c
