From 1459df2cab9a7ee861c6ccd16e618ed1f0808a28 Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Mon, 2 May 2022 17:07:40 -0600 Subject: [PATCH 1/7] gh-92203: Add closure support to exec(). --- Doc/library/functions.rst | 10 +++++- Lib/test/test_builtin.py | 44 ++++++++++++++++++++++++- Python/bltinmodule.c | 60 ++++++++++++++++++++++++++++++----- Python/clinic/bltinmodule.c.h | 37 ++++++++++++++------- 4 files changed, 130 insertions(+), 21 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index cb862968970825..f9f76ac766a2b7 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -496,7 +496,7 @@ are always available. They are listed here in alphabetical order. n += 1 -.. function:: eval(expression[, globals[, locals]]) +.. function:: eval(expression[, globals[, locals]], *, closure=None) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping @@ -576,6 +576,11 @@ are always available. They are listed here in alphabetical order. builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + The *closure* argument specifies a closure--a tuple of cellvars. + It's only valid when the *object* is a code object containing free variables. + The length of the tuple must exactly match the number of free variables + referenced by the code object. + .. audit-event:: exec code_object exec Raises an :ref:`auditing event ` ``exec`` with the code object @@ -594,6 +599,9 @@ are always available. They are listed here in alphabetical order. Pass an explicit *locals* dictionary if you need to see effects of the code on *locals* after function :func:`exec` returns. + .. versionchanged:: 3.11 + Added the *closure* parameter. + .. function:: filter(function, iterable) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 29039230201aca..8f90adac154314 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -24,7 +24,7 @@ from inspect import CO_COROUTINE from itertools import product from textwrap import dedent -from types import AsyncGeneratorType, FunctionType +from types import AsyncGeneratorType, FunctionType, CellType from operator import neg from test import support from test.support import (swap_attr, maybe_get_event_loop_policy) @@ -772,6 +772,48 @@ def test_exec_redirected(self): finally: sys.stdout = savestdout + def test_exec_closure(self): + result = 0 + def make_closure_functions(): + a = 2 + b = 3 + c = 5 + def two_freevars(): + nonlocal result + nonlocal a + nonlocal b + result = a*b + def three_freevars(): + nonlocal result + nonlocal a + nonlocal b + nonlocal c + result = a*b*c + return two_freevars, three_freevars + two_freevars, three_freevars = make_closure_functions() + + exec(two_freevars.__code__, + two_freevars.__globals__, + closure=two_freevars.__closure__) + self.assertEquals(result, 6) + + result = 0 + my_closure = list(two_freevars.__closure__) + my_closure[0] = CellType(35) + my_closure[1] = CellType(72) + my_closure = tuple(my_closure) + exec(two_freevars.__code__, + two_freevars.__globals__, + closure=my_closure) + self.assertEquals(result, 2520) + + self.assertRaises(ValueError, + exec, + two_freevars.__code__, + two_freevars.__globals__, + closure=three_freevars.__closure__) + + def test_filter(self): self.assertEqual(list(filter(lambda c: 'a' <= c <= 'z', 'Hello World')), list('elloorld')) self.assertEqual(list(filter(None, [1, 'hello', [], [3], '', None, 9, 0])), [1, 'hello', [3], 9]) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 84ebb680e0b8fb..17d4420f5f3039 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -977,6 +977,8 @@ exec as builtin_exec globals: object = None locals: object = None / + * + closure: object(c_default="NULL") = None Execute the given source in the context of globals and locals. @@ -985,12 +987,14 @@ or a code object as returned by compile(). The globals must be a dictionary and locals can be any mapping, defaulting to the current globals and locals. If only globals is given, locals defaults to it. +The closure must be a tuple of cellvars, and can only be used +when source is a code object requiring exactly that many cellvars. [clinic start generated code]*/ static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, - PyObject *locals) -/*[clinic end generated code: output=3c90efc6ab68ef5d input=01ca3e1c01692829]*/ + PyObject *locals, PyObject *closure) +/*[clinic end generated code: output=7579eb4e7646743d input=9fb91ea3cb24f475]*/ { PyObject *v; @@ -1029,20 +1033,60 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, return NULL; } + if (closure == Py_None) { + closure = NULL; + } + if (PyCode_Check(source)) { + Py_ssize_t num_free = PyCode_GetNumFree((PyCodeObject *)source); + if (num_free == 0) { + if (closure) { + PyErr_SetString(PyExc_ValueError, + "code object cannot use a closure"); + return NULL; + } + } else { + int closure_is_ok = + closure + && PyTuple_CheckExact(closure) + && (PyTuple_GET_SIZE(closure) == num_free); + if (closure_is_ok) { + for (Py_ssize_t i = 0; i < num_free; i++) { + PyObject *cell = PyTuple_GET_ITEM(closure, i); + if (!PyCell_Check(cell)) { + closure_is_ok = 0; + break; + } + } + } + if (!closure_is_ok) { + PyErr_Format(PyExc_TypeError, + "code object requires a closure of exactly length %d", + num_free); + return NULL; + } + } + if (PySys_Audit("exec", "O", source) < 0) { return NULL; } - if (PyCode_GetNumFree((PyCodeObject *)source) > 0) { - PyErr_SetString(PyExc_TypeError, - "code object passed to exec() may not " - "contain free variables"); - return NULL; + if (!closure) { + v = PyEval_EvalCode(source, globals, locals); + } else { + v = PyEval_EvalCodeEx(source, globals, locals, + NULL, 0, + NULL, 0, + NULL, 0, + NULL, + closure); } - v = PyEval_EvalCode(source, globals, locals); } else { + if (closure != NULL) { + PyErr_SetString(PyExc_ValueError, + "closure cannot be used when source is a string"); + } PyObject *source_copy; const char *str; PyCompilerFlags cf = _PyCompilerFlags_INIT; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 4053f5a341e1fe..45e5ddaba388f2 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -408,7 +408,7 @@ builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_exec__doc__, -"exec($module, source, globals=None, locals=None, /)\n" +"exec($module, source, globals=None, locals=None, /, *, closure=None)\n" "--\n" "\n" "Execute the given source in the context of globals and locals.\n" @@ -417,37 +417,52 @@ PyDoc_STRVAR(builtin_exec__doc__, "or a code object as returned by compile().\n" "The globals must be a dictionary and locals can be any mapping,\n" "defaulting to the current globals and locals.\n" -"If only globals is given, locals defaults to it."); +"If only globals is given, locals defaults to it.\n" +"The closure must be a tuple of cellvars, and can only be used\n" +"when source is a code object."); #define BUILTIN_EXEC_METHODDEF \ - {"exec", (PyCFunction)(void(*)(void))builtin_exec, METH_FASTCALL, builtin_exec__doc__}, + {"exec", (PyCFunction)(void(*)(void))builtin_exec, METH_FASTCALL|METH_KEYWORDS, builtin_exec__doc__}, static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, - PyObject *locals); + PyObject *locals, PyObject *closure); static PyObject * -builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "", "closure", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "exec", 0}; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *source; PyObject *globals = Py_None; PyObject *locals = Py_None; + PyObject *closure = NULL; - if (!_PyArg_CheckPositional("exec", nargs, 1, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { goto exit; } source = args[0]; if (nargs < 2) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; globals = args[1]; if (nargs < 3) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; locals = args[2]; -skip_optional: - return_value = builtin_exec_impl(module, source, globals, locals); +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + closure = args[3]; +skip_optional_kwonly: + return_value = builtin_exec_impl(module, source, globals, locals, closure); exit: return return_value; @@ -1030,4 +1045,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d341fa7525f30070 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=02fca5efe8a1b52a input=a9049054013a1b77]*/ From 8db39e88acef03080c8670be0f82785441b4d2d2 Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Mon, 2 May 2022 17:12:54 -0600 Subject: [PATCH 2/7] Add blurb. --- .../2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst new file mode 100644 index 00000000000000..f765579a1b627e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-02-17-12-49.gh-issue-92203.-igcjS.rst @@ -0,0 +1,5 @@ +Add a closure keyword-only parameter to exec(). It can only be specified +when exec-ing a code object that uses free variables. When specified, it +must be a tuple, with exactly the number of cell variables referenced by the +code object. closure has a default value of None, and it must be None if the +code object doesn't refer to any free variables. From 6df4ce27841029b9dd2f6bc679838314d6ad5520 Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Mon, 2 May 2022 18:04:57 -0600 Subject: [PATCH 3/7] Fix the broken test. I changed the exception raised to match the exception raised by existing code and neglected to update the corresponding test. So... the test worked! And now, also, it passes. --- Lib/test/test_builtin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 8f90adac154314..130f33d1c602ac 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -807,7 +807,7 @@ def three_freevars(): closure=my_closure) self.assertEquals(result, 2520) - self.assertRaises(ValueError, + self.assertRaises(TypeError, exec, two_freevars.__code__, two_freevars.__globals__, From 66aa6703946693b7357672d25a2bfe27b4a8babb Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Mon, 2 May 2022 23:37:18 -0600 Subject: [PATCH 4/7] Regen'd clinic, as I changed the docstring. --- Python/bltinmodule.c | 2 +- Python/clinic/bltinmodule.c.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 17d4420f5f3039..5b998b15272441 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -994,7 +994,7 @@ when source is a code object requiring exactly that many cellvars. static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals, PyObject *closure) -/*[clinic end generated code: output=7579eb4e7646743d input=9fb91ea3cb24f475]*/ +/*[clinic end generated code: output=7579eb4e7646743d input=f13a7e2b503d1d9a]*/ { PyObject *v; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 45e5ddaba388f2..8c822679922dc7 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -419,7 +419,7 @@ PyDoc_STRVAR(builtin_exec__doc__, "defaulting to the current globals and locals.\n" "If only globals is given, locals defaults to it.\n" "The closure must be a tuple of cellvars, and can only be used\n" -"when source is a code object."); +"when source is a code object requiring exactly that many cellvars."); #define BUILTIN_EXEC_METHODDEF \ {"exec", (PyCFunction)(void(*)(void))builtin_exec, METH_FASTCALL|METH_KEYWORDS, builtin_exec__doc__}, @@ -1045,4 +1045,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=02fca5efe8a1b52a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=82a03162d11a826b input=a9049054013a1b77]*/ From 9abf4506077cdd364ce7cfd02804e85a57b3fa71 Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Thu, 5 May 2022 23:55:36 -0700 Subject: [PATCH 5/7] Changes from reviews. Thanks, Serhiy and Jelle! --- Doc/library/functions.rst | 4 ++-- Lib/test/test_builtin.py | 48 +++++++++++++++++++++++++++------------ Python/bltinmodule.c | 4 ++-- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index f9f76ac766a2b7..ba81cc954e7b6d 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -496,7 +496,7 @@ are always available. They are listed here in alphabetical order. n += 1 -.. function:: eval(expression[, globals[, locals]], *, closure=None) +.. function:: eval(expression[, globals[, locals]]) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping @@ -547,7 +547,7 @@ are always available. They are listed here in alphabetical order. .. index:: builtin: exec -.. function:: exec(object[, globals[, locals]]) +.. function:: exec(object[, globals[, locals]], *, closure=None) This function supports dynamic execution of Python code. *object* must be either a string or a code object. If it is a string, the string is parsed as diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 130f33d1c602ac..60fef20c7458f2 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -778,40 +778,58 @@ def make_closure_functions(): a = 2 b = 3 c = 5 - def two_freevars(): + def three_freevars(): nonlocal result nonlocal a nonlocal b result = a*b - def three_freevars(): + def four_freevars(): nonlocal result nonlocal a nonlocal b nonlocal c result = a*b*c - return two_freevars, three_freevars - two_freevars, three_freevars = make_closure_functions() + return three_freevars, four_freevars + three_freevars, four_freevars = make_closure_functions() - exec(two_freevars.__code__, - two_freevars.__globals__, - closure=two_freevars.__closure__) - self.assertEquals(result, 6) + exec(three_freevars.__code__, + three_freevars.__globals__, + closure=three_freevars.__closure__) + self.assertEqual(result, 6) result = 0 - my_closure = list(two_freevars.__closure__) + my_closure = list(three_freevars.__closure__) my_closure[0] = CellType(35) my_closure[1] = CellType(72) my_closure = tuple(my_closure) - exec(two_freevars.__code__, - two_freevars.__globals__, + exec(three_freevars.__code__, + three_freevars.__globals__, closure=my_closure) - self.assertEquals(result, 2520) + self.assertEqual(result, 2520) + # test with tuple of wrong length self.assertRaises(TypeError, exec, - two_freevars.__code__, - two_freevars.__globals__, - closure=three_freevars.__closure__) + three_freevars.__code__, + three_freevars.__globals__, + closure=four_freevars.__closure__) + + # test with list instead of tuple + my_closure = list(my_closure) + self.assertRaises(TypeError, + exec, + three_freevars.__code__, + three_freevars.__globals__, + closure=my_closure) + + # test with non-cellvar in tuple + my_closure[0] = int + my_closure = tuple(my_closure) + self.assertRaises(TypeError, + exec, + three_freevars.__code__, + three_freevars.__globals__, + closure=my_closure) def test_filter(self): diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5b998b15272441..fa0152fb5a661d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1084,8 +1084,8 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } else { if (closure != NULL) { - PyErr_SetString(PyExc_ValueError, - "closure cannot be used when source is a string"); + PyErr_SetString(PyExc_TypeError, + "closure can only be used when source is a code object"); } PyObject *source_copy; const char *str; From c85c2d3c564e8facf60653646d47a0a228a8a91a Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Fri, 6 May 2022 01:27:27 -0700 Subject: [PATCH 6/7] Fix another exception, add another test. --- Lib/test/test_builtin.py | 32 +++++++++++++++++++++++++------- Python/bltinmodule.c | 4 ++-- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 60fef20c7458f2..ba7a7e20d7dcda 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -773,6 +773,9 @@ def test_exec_redirected(self): sys.stdout = savestdout def test_exec_closure(self): + def function_without_closures(): + return 3 * 5 + result = 0 def make_closure_functions(): a = 2 @@ -792,29 +795,44 @@ def four_freevars(): return three_freevars, four_freevars three_freevars, four_freevars = make_closure_functions() + # "smoke" test + result = 0 exec(three_freevars.__code__, three_freevars.__globals__, closure=three_freevars.__closure__) self.assertEqual(result, 6) + # should also work with a manually created closure result = 0 - my_closure = list(three_freevars.__closure__) - my_closure[0] = CellType(35) - my_closure[1] = CellType(72) - my_closure = tuple(my_closure) + my_closure = (CellType(35), CellType(72), three_freevars.__closure__[2]) exec(three_freevars.__code__, three_freevars.__globals__, closure=my_closure) self.assertEqual(result, 2520) - # test with tuple of wrong length + # should fail: closure isn't allowed + # for functions without free vars + self.assertRaises(TypeError, + exec, + function_without_closures.__code__, + function_without_closures.__globals__, + closure=my_closure) + + # should fail: closure required but wasn't specified + self.assertRaises(TypeError, + exec, + three_freevars.__code__, + three_freevars.__globals__, + closure=None) + + # should fail: closure of wrong length self.assertRaises(TypeError, exec, three_freevars.__code__, three_freevars.__globals__, closure=four_freevars.__closure__) - # test with list instead of tuple + # should fail: closure using a list instead of a tuple my_closure = list(my_closure) self.assertRaises(TypeError, exec, @@ -822,7 +840,7 @@ def four_freevars(): three_freevars.__globals__, closure=my_closure) - # test with non-cellvar in tuple + # should fail: closure tuple with one non-cell-var my_closure[0] = int my_closure = tuple(my_closure) self.assertRaises(TypeError, diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index fa0152fb5a661d..5b285e98ed639a 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1041,8 +1041,8 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, Py_ssize_t num_free = PyCode_GetNumFree((PyCodeObject *)source); if (num_free == 0) { if (closure) { - PyErr_SetString(PyExc_ValueError, - "code object cannot use a closure"); + PyErr_SetString(PyExc_TypeError, + "cannot use a closure with this code object"); return NULL; } } else { From 24b8664c8b5c1efe8ce9515a37540b1ae1782ccf Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Fri, 6 May 2022 02:41:17 -0700 Subject: [PATCH 7/7] Changed formatter to %zd. --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5b285e98ed639a..072bf75bf8d697 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1061,7 +1061,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } if (!closure_is_ok) { PyErr_Format(PyExc_TypeError, - "code object requires a closure of exactly length %d", + "code object requires a closure of exactly length %zd", num_free); return NULL; } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy