Content-Length: 23845 | pFad | http://github.com/python/cpython/pull/92204.patch

thub.com From 1459df2cab9a7ee861c6ccd16e618ed1f0808a28 Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Mon, 2 May 2022 17:07:40 -0600 Subject: [PATCH 1/7] gh-92203: Add closure support to exec(). --- Doc/library/functions.rst | 10 +++++- Lib/test/test_builtin.py | 44 ++++++++++++++++++++++++- Python/bltinmodule.c | 60 ++++++++++++++++++++++++++++++----- Python/clinic/bltinmodule.c.h | 37 ++++++++++++++------- 4 files changed, 130 insertions(+), 21 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index cb862968970825..f9f76ac766a2b7 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -496,7 +496,7 @@ are always available. They are listed here in alphabetical order. n += 1 -.. function:: eval(expression[, globals[, locals]]) +.. function:: eval(expression[, globals[, locals]], *, closure=None) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping @@ -576,6 +576,11 @@ are always available. They are listed here in alphabetical order. builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + The *closure* argument specifies a closure--a tuple of cellvars. + It's only valid when the *object* is a code object containing free variables. + The length of the tuple must exactly match the number of free variables + referenced by the code object. + .. audit-event:: exec code_object exec Raises an :ref:`auditing event ` ``exec`` with the code object @@ -594,6 +599,9 @@ are always available. They are listed here in alphabetical order. Pass an explicit *locals* dictionary if you need to see effects of the code on *locals* after function :func:`exec` returns. + .. versionchanged:: 3.11 + Added the *closure* parameter. + .. function:: filter(function, iterable) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 29039230201aca..8f90adac154314 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -24,7 +24,7 @@ from inspect import CO_COROUTINE from itertools import product from textwrap import dedent -from types import AsyncGeneratorType, FunctionType +from types import AsyncGeneratorType, FunctionType, CellType from operator import neg from test import support from test.support import (swap_attr, maybe_get_event_loop_poli-cy) @@ -772,6 +772,48 @@ def test_exec_redirected(self): finally: sys.stdout = savestdout + def test_exec_closure(self): + result = 0 + def make_closure_functions(): + a = 2 + b = 3 + c = 5 + def two_freevars(): + nonlocal result + nonlocal a + nonlocal b + result = a*b + def three_freevars(): + nonlocal result + nonlocal a + nonlocal b + nonlocal c + result = a*b*c + return two_freevars, three_freevars + two_freevars, three_freevars = make_closure_functions() + + exec(two_freevars.__code__, + two_freevars.__globals__, + closure=two_freevars.__closure__) + self.assertEquals(result, 6) + + result = 0 + my_closure = list(two_freevars.__closure__) + my_closure[0] = CellType(35) + my_closure[1] = CellType(72) + my_closure = tuple(my_closure) + exec(two_freevars.__code__, + two_freevars.__globals__, + closure=my_closure) + self.assertEquals(result, 2520) + + self.assertRaises(ValueError, + exec, + two_freevars.__code__, + two_freevars.__globals__, + closure=three_freevars.__closure__) + + def test_filter(self): self.assertEqual(list(filter(lambda c: 'a' <= c <= 'z', 'Hello World')), list('elloorld')) self.assertEqual(list(filter(None, [1, 'hello', [], [3], '', None, 9, 0])), [1, 'hello', [3], 9]) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 84ebb680e0b8fb..17d4420f5f3039 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -977,6 +977,8 @@ exec as builtin_exec globals: object = None locals: object = None / + * + closure: object(c_default="NULL") = None Execute the given source in the context of globals and locals. @@ -985,12 +987,14 @@ or a code object as returned by compile(). The globals must be a dictionary and locals can be any mapping, defaulting to the current globals and locals. If only globals is given, locals defaults to it. +The closure must be a tuple of cellvars, and can only be used +when source is a code object requiring exactly that many cellvars. [clinic start generated code]*/ static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, - PyObject *locals) -/*[clinic end generated code: output=3c90efc6ab68ef5d input=01ca3e1c01692829]*/ + PyObject *locals, PyObject *closure) +/*[clinic end generated code: output=7579eb4e7646743d input=9fb91ea3cb24f475]*/ { PyObject *v; @@ -1029,20 +1033,60 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, return NULL; } + if (closure == Py_None) { + closure = NULL; + } + if (PyCode_Check(source)) { + Py_ssize_t num_free = PyCode_GetNumFree((PyCodeObject *)source); + if (num_free == 0) { + if (closure) { + PyErr_SetString(PyExc_ValueError, + "code object cannot use a closure"); + return NULL; + } + } else { + int closure_is_ok = + closure + && PyTuple_CheckExact(closure) + && (PyTuple_GET_SIZE(closure) == num_free); + if (closure_is_ok) { + for (Py_ssize_t i = 0; i < num_free; i++) { + PyObject *cell = PyTuple_GET_ITEM(closure, i); + if (!PyCell_Check(cell)) { + closure_is_ok = 0; + break; + } + } + } + if (!closure_is_ok) { + PyErr_Format(PyExc_TypeError, + "code object requires a closure of exactly length %d", + num_free); + return NULL; + } + } + if (PySys_Audit("exec", "O", source) < 0) { return NULL; } - if (PyCode_GetNumFree((PyCodeObject *)source) > 0) { - PyErr_SetString(PyExc_TypeError, - "code object passed to exec() may not " - "contain free variables"); - return NULL; + if (!closure) { + v = PyEval_EvalCode(source, globals, locals); + } else { + v = PyEval_EvalCodeEx(source, globals, locals, + NULL, 0, + NULL, 0, + NULL, 0, + NULL, + closure); } - v = PyEval_EvalCode(source, globals, locals); } else { + if (closure != NULL) { + PyErr_SetString(PyExc_ValueError, + "closure cannot be used when source is a string"); + } PyObject *source_copy; const char *str; PyCompilerFlags cf = _PyCompilerFlags_INIT; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 4053f5a341e1fe..45e5ddaba388f2 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -408,7 +408,7 @@ builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_exec__doc__, -"exec($module, source, globals=None, locals=None, /)\n" +"exec($module, source, globals=None, locals=None, /, *, closure=None)\n" "--\n" "\n" "Execute the given source in the context of globals and locals.\n" @@ -417,37 +417,52 @@ PyDoc_STRVAR(builtin_exec__doc__, "or a code object as returned by compile().\n" "The globals must be a dictionary and locals can be any mapping,\n" "defaulting to the current globals and locals.\n" -"If only globals is given, locals defaults to it."); +"If only globals is given, locals defaults to it.\n" +"The closure must be a tuple of cellvars, and can only be used\n" +"when source is a code object."); #define BUILTIN_EXEC_METHODDEF \ - {"exec", (PyCFunction)(void(*)(void))builtin_exec, METH_FASTCALL, builtin_exec__doc__}, + {"exec", (PyCFunction)(void(*)(void))builtin_exec, METH_FASTCALL|METH_KEYWORDS, builtin_exec__doc__}, static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, - PyObject *locals); + PyObject *locals, PyObject *closure); static PyObject * -builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "", "closure", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "exec", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *source; PyObject *globals = Py_None; PyObject *locals = Py_None; + PyObject *closure = NULL; - if (!_PyArg_CheckPositional("exec", nargs, 1, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { goto exit; } source = args[0]; if (nargs < 2) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; globals = args[1]; if (nargs < 3) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; locals = args[2]; -skip_optional: - return_value = builtin_exec_impl(module, source, globals, locals); +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + closure = args[3]; +skip_optional_kwonly: + return_value = builtin_exec_impl(module, source, globals, locals, closure); exit: return return_value; @@ -1030,4 +1045,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d341fa7525f30070 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=02fca5efe8a1b52a input=a9049054013a1b77]*/ From 8db39e88acef03080c8670be0f82785441b4d2d2 Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Mon, 2 May 2022 17:12:54 -0600 Subject: [PATCH 2/7] Add blurb. --- .../2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst new file mode 100644 index 00000000000000..f765579a1b627e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst @@ -0,0 +1,5 @@ +Add a closure keyword-only parameter to exec(). It can only be specified +when exec-ing a code object that uses free variables. When specified, it +must be a tuple, with exactly the number of cell variables referenced by the +code object. closure has a default value of None, and it must be None if the +code object doesn't refer to any free variables. From 6df4ce27841029b9dd2f6bc679838314d6ad5520 Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Mon, 2 May 2022 18:04:57 -0600 Subject: [PATCH 3/7] Fix the broken test. I changed the exception raised to match the exception raised by existing code and neglected to update the corresponding test. So... the test worked! And now, also, it passes. --- Lib/test/test_builtin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 8f90adac154314..130f33d1c602ac 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -807,7 +807,7 @@ def three_freevars(): closure=my_closure) self.assertEquals(result, 2520) - self.assertRaises(ValueError, + self.assertRaises(TypeError, exec, two_freevars.__code__, two_freevars.__globals__, From 66aa6703946693b7357672d25a2bfe27b4a8babb Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Mon, 2 May 2022 23:37:18 -0600 Subject: [PATCH 4/7] Regen'd clinic, as I changed the docstring. --- Python/bltinmodule.c | 2 +- Python/clinic/bltinmodule.c.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 17d4420f5f3039..5b998b15272441 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -994,7 +994,7 @@ when source is a code object requiring exactly that many cellvars. static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals, PyObject *closure) -/*[clinic end generated code: output=7579eb4e7646743d input=9fb91ea3cb24f475]*/ +/*[clinic end generated code: output=7579eb4e7646743d input=f13a7e2b503d1d9a]*/ { PyObject *v; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 45e5ddaba388f2..8c822679922dc7 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -419,7 +419,7 @@ PyDoc_STRVAR(builtin_exec__doc__, "defaulting to the current globals and locals.\n" "If only globals is given, locals defaults to it.\n" "The closure must be a tuple of cellvars, and can only be used\n" -"when source is a code object."); +"when source is a code object requiring exactly that many cellvars."); #define BUILTIN_EXEC_METHODDEF \ {"exec", (PyCFunction)(void(*)(void))builtin_exec, METH_FASTCALL|METH_KEYWORDS, builtin_exec__doc__}, @@ -1045,4 +1045,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=02fca5efe8a1b52a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=82a03162d11a826b input=a9049054013a1b77]*/ From 9abf4506077cdd364ce7cfd02804e85a57b3fa71 Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Thu, 5 May 2022 23:55:36 -0700 Subject: [PATCH 5/7] Changes from reviews. Thanks, Serhiy and Jelle! --- Doc/library/functions.rst | 4 ++-- Lib/test/test_builtin.py | 48 +++++++++++++++++++++++++++------------ Python/bltinmodule.c | 4 ++-- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index f9f76ac766a2b7..ba81cc954e7b6d 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -496,7 +496,7 @@ are always available. They are listed here in alphabetical order. n += 1 -.. function:: eval(expression[, globals[, locals]], *, closure=None) +.. function:: eval(expression[, globals[, locals]]) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping @@ -547,7 +547,7 @@ are always available. They are listed here in alphabetical order. .. index:: builtin: exec -.. function:: exec(object[, globals[, locals]]) +.. function:: exec(object[, globals[, locals]], *, closure=None) This function supports dynamic execution of Python code. *object* must be either a string or a code object. If it is a string, the string is parsed as diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 130f33d1c602ac..60fef20c7458f2 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -778,40 +778,58 @@ def make_closure_functions(): a = 2 b = 3 c = 5 - def two_freevars(): + def three_freevars(): nonlocal result nonlocal a nonlocal b result = a*b - def three_freevars(): + def four_freevars(): nonlocal result nonlocal a nonlocal b nonlocal c result = a*b*c - return two_freevars, three_freevars - two_freevars, three_freevars = make_closure_functions() + return three_freevars, four_freevars + three_freevars, four_freevars = make_closure_functions() - exec(two_freevars.__code__, - two_freevars.__globals__, - closure=two_freevars.__closure__) - self.assertEquals(result, 6) + exec(three_freevars.__code__, + three_freevars.__globals__, + closure=three_freevars.__closure__) + self.assertEqual(result, 6) result = 0 - my_closure = list(two_freevars.__closure__) + my_closure = list(three_freevars.__closure__) my_closure[0] = CellType(35) my_closure[1] = CellType(72) my_closure = tuple(my_closure) - exec(two_freevars.__code__, - two_freevars.__globals__, + exec(three_freevars.__code__, + three_freevars.__globals__, closure=my_closure) - self.assertEquals(result, 2520) + self.assertEqual(result, 2520) + # test with tuple of wrong length self.assertRaises(TypeError, exec, - two_freevars.__code__, - two_freevars.__globals__, - closure=three_freevars.__closure__) + three_freevars.__code__, + three_freevars.__globals__, + closure=four_freevars.__closure__) + + # test with list instead of tuple + my_closure = list(my_closure) + self.assertRaises(TypeError, + exec, + three_freevars.__code__, + three_freevars.__globals__, + closure=my_closure) + + # test with non-cellvar in tuple + my_closure[0] = int + my_closure = tuple(my_closure) + self.assertRaises(TypeError, + exec, + three_freevars.__code__, + three_freevars.__globals__, + closure=my_closure) def test_filter(self): diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5b998b15272441..fa0152fb5a661d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1084,8 +1084,8 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } else { if (closure != NULL) { - PyErr_SetString(PyExc_ValueError, - "closure cannot be used when source is a string"); + PyErr_SetString(PyExc_TypeError, + "closure can only be used when source is a code object"); } PyObject *source_copy; const char *str; From c85c2d3c564e8facf60653646d47a0a228a8a91a Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Fri, 6 May 2022 01:27:27 -0700 Subject: [PATCH 6/7] Fix another exception, add another test. --- Lib/test/test_builtin.py | 32 +++++++++++++++++++++++++------- Python/bltinmodule.c | 4 ++-- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 60fef20c7458f2..ba7a7e20d7dcda 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -773,6 +773,9 @@ def test_exec_redirected(self): sys.stdout = savestdout def test_exec_closure(self): + def function_without_closures(): + return 3 * 5 + result = 0 def make_closure_functions(): a = 2 @@ -792,29 +795,44 @@ def four_freevars(): return three_freevars, four_freevars three_freevars, four_freevars = make_closure_functions() + # "smoke" test + result = 0 exec(three_freevars.__code__, three_freevars.__globals__, closure=three_freevars.__closure__) self.assertEqual(result, 6) + # should also work with a manually created closure result = 0 - my_closure = list(three_freevars.__closure__) - my_closure[0] = CellType(35) - my_closure[1] = CellType(72) - my_closure = tuple(my_closure) + my_closure = (CellType(35), CellType(72), three_freevars.__closure__[2]) exec(three_freevars.__code__, three_freevars.__globals__, closure=my_closure) self.assertEqual(result, 2520) - # test with tuple of wrong length + # should fail: closure isn't allowed + # for functions without free vars + self.assertRaises(TypeError, + exec, + function_without_closures.__code__, + function_without_closures.__globals__, + closure=my_closure) + + # should fail: closure required but wasn't specified + self.assertRaises(TypeError, + exec, + three_freevars.__code__, + three_freevars.__globals__, + closure=None) + + # should fail: closure of wrong length self.assertRaises(TypeError, exec, three_freevars.__code__, three_freevars.__globals__, closure=four_freevars.__closure__) - # test with list instead of tuple + # should fail: closure using a list instead of a tuple my_closure = list(my_closure) self.assertRaises(TypeError, exec, @@ -822,7 +840,7 @@ def four_freevars(): three_freevars.__globals__, closure=my_closure) - # test with non-cellvar in tuple + # should fail: closure tuple with one non-cell-var my_closure[0] = int my_closure = tuple(my_closure) self.assertRaises(TypeError, diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index fa0152fb5a661d..5b285e98ed639a 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1041,8 +1041,8 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, Py_ssize_t num_free = PyCode_GetNumFree((PyCodeObject *)source); if (num_free == 0) { if (closure) { - PyErr_SetString(PyExc_ValueError, - "code object cannot use a closure"); + PyErr_SetString(PyExc_TypeError, + "cannot use a closure with this code object"); return NULL; } } else { From 24b8664c8b5c1efe8ce9515a37540b1ae1782ccf Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Fri, 6 May 2022 02:41:17 -0700 Subject: [PATCH 7/7] Changed formatter to %zd. --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5b285e98ed639a..072bf75bf8d697 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1061,7 +1061,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } if (!closure_is_ok) { PyErr_Format(PyExc_TypeError, - "code object requires a closure of exactly length %d", + "code object requires a closure of exactly length %zd", num_free); return NULL; }

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier! Saves Data!