Skip to content

gh-132661: Disallow Template/str concatenation after PEP 750 spec update #135996

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 27 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
5d5e187
Disallow explicit concat between Template and str
davepeck Jun 26, 2025
1cc778f
First pass at removing implicit Template/str concat
davepeck Jun 26, 2025
7ea498c
Merge branch 'main' into pep750-concat-update
davepeck Jun 30, 2025
d08c8b8
Merge branch 'main' into pep750-concat-update
davepeck Jul 8, 2025
c662041
Remove extraneous newline
davepeck Jul 8, 2025
b17b7c9
Add expected exception messages to tests
davepeck Jul 8, 2025
a0c1bb6
Remove _ast_unparse t/f implicit concat helper code
davepeck Jul 8, 2025
23988e8
Merge branch 'main' into pep750-concat-update
davepeck Jul 8, 2025
d7eadec
Add a parallel comment
davepeck Jul 8, 2025
cb1ad63
Add news blurb for PR.
davepeck Jul 8, 2025
741eaf2
One day, I may develop RST muscle memory.
davepeck Jul 8, 2025
8aeb512
Merge branch 'main' into pep750-concat-update
davepeck Jul 9, 2025
bf5447b
Merge branch 'main' into pep750-concat-update
davepeck Jul 9, 2025
7e7ea5a
Merge branch 'main' into pep750-concat-update
davepeck Jul 9, 2025
2f79337
Two minor tweaks
davepeck Jul 9, 2025
dacafdb
Use subtests; test both directions.
davepeck Jul 9, 2025
b272287
Update Objects/templateobject.c
davepeck Jul 9, 2025
ffae8e9
Update Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issu…
davepeck Jul 9, 2025
c6ca45a
Fix bug in suggested change
davepeck Jul 9, 2025
091a2a9
Merge branch 'main' into pep750-concat-update
davepeck Jul 9, 2025
860caea
First pass at update to grammar and action_helpers.c
davepeck Jul 10, 2025
6a6f734
Merge branch 'main' into pep750-concat-update
davepeck Jul 10, 2025
860f81d
Simplify grammar
davepeck Jul 10, 2025
e0fe608
Use KNOWN_RANGE for string/t-string mix error messages
davepeck Jul 10, 2025
c9ed06b
Merge branch 'pep750-concat-update' of github.com:t-strings/cpython i…
davepeck Jul 10, 2025
3ebdd97
Merge branch 'main' into pep750-concat-update
davepeck Jul 10, 2025
ef1cd5c
Clean up ast_unparse.c append_tstring()
davepeck Jul 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion Grammar/python.gram
Original file line number Diff line number Diff line change
Expand Up @@ -987,7 +987,10 @@ tstring[expr_ty] (memo):
_PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) }

string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
strings[expr_ty] (memo):
| invalid_string_tstring_concat
| a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
| a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) }

list[expr_ty]:
| '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }
Expand Down Expand Up @@ -1553,6 +1556,12 @@ invalid_tstring_conversion_character:
| '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
| '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }

invalid_string_tstring_concat:
| a=(fstring|string)+ b[expr_ty]=tstring {
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-strings with strings or f-strings") }
| a=tstring+ b[expr_ty]=(fstring|string) {
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-strings with strings or f-strings") }

invalid_arithmetic:
| sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
invalid_factor:
Expand Down
26 changes: 1 addition & 25 deletions Lib/_ast_unparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -626,35 +626,11 @@ def _write_ftstring(self, values, prefix):
)
self._ftstring_helper(fstring_parts)

def _tstring_helper(self, node):
if not node.values:
self._write_ftstring([], "t")
return
last_idx = 0
for i, value in enumerate(node.values):
# This can happen if we have an implicit concat of a t-string
# with an f-string
if isinstance(value, FormattedValue):
if i > last_idx:
# Write t-string until here
self._write_ftstring(node.values[last_idx:i], "t")
self.write(" ")
# Write f-string with the current formatted value
self._write_ftstring([node.values[i]], "f")
if i + 1 < len(node.values):
# Only add a space if there are more values after this
self.write(" ")
last_idx = i + 1

if last_idx < len(node.values):
# Write t-string from last_idx to end
self._write_ftstring(node.values[last_idx:], "t")

def visit_JoinedStr(self, node):
self._write_ftstring(node.values, "f")

def visit_TemplateStr(self, node):
self._tstring_helper(node)
self._write_ftstring(node.values, "t")

def _write_ftstring_inner(self, node, is_format_spec=False):
if isinstance(node, JoinedStr):
Expand Down
7 changes: 0 additions & 7 deletions Lib/test/test_ast/test_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -999,13 +999,6 @@ def test_tstring(self):
self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)

# Test AST for implicit concat of t-string with f-string
tree = ast.parse('t"Hello {name}" f"{name}"')
self.assertIsInstance(tree.body[0].value, ast.TemplateStr)
self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue)


class CopyTests(unittest.TestCase):
"""Test copying and pickling AST nodes."""
Expand Down
71 changes: 23 additions & 48 deletions Lib/test/test_tstring.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,6 @@ def test_raw_tstrings(self):
t = tr"{path}\Documents"
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])


def test_template_concatenation(self):
# Test template + template
t1 = t"Hello, "
Expand All @@ -161,9 +160,9 @@ def test_template_concatenation(self):

# Test template + string
t1 = t"Hello"
combined = t1 + ", world"
self.assertTStringEqual(combined, ("Hello, world",), ())
self.assertEqual(fstring(combined), "Hello, world")
expected_msg = 'can only concatenate Template (not "str") to Template'
with self.assertRaises(TypeError, msg=expected_msg):
t1 + ", world"

# Test template + template with interpolation
name = "Python"
Expand All @@ -174,9 +173,9 @@ def test_template_concatenation(self):
self.assertEqual(fstring(combined), "Hello, Python")

# Test string + template
t = "Hello, " + t"{name}"
self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
self.assertEqual(fstring(t), "Hello, Python")
expected_msg = 'can only concatenate str (not "string.templatelib.Template") to str'
with self.assertRaises(TypeError, msg=expected_msg):
"Hello, " + t"{name}"

def test_nested_templates(self):
# Test a template inside another template expression
Expand Down Expand Up @@ -241,52 +240,28 @@ def test_literal_concatenation(self):
self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
self.assertEqual(fstring(t), "Hello, Python")

# Test concatenation with string literal
name = "Python"
t = t"Hello, {name}" "and welcome!"
self.assertTStringEqual(
t, ("Hello, ", "and welcome!"), [(name, "name")]
)
self.assertEqual(fstring(t), "Hello, Pythonand welcome!")

# Test concatenation with Unicode literal
name = "Python"
t = t"Hello, {name}" u"and welcome!"
self.assertTStringEqual(
t, ("Hello, ", "and welcome!"), [(name, "name")]
)
self.assertEqual(fstring(t), "Hello, Pythonand welcome!")

# Test concatenation with f-string literal
tab = '\t'
t = t"Tab: {tab}. " f"f-tab: {tab}."
self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")])
self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.")

# Test concatenation with raw string literal
tab = '\t'
t = t"Tab: {tab}. " r"Raw tab: \t."
self.assertTStringEqual(
t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")]
)
self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.")

# Test concatenation with raw f-string literal
tab = '\t'
t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t."
self.assertTStringEqual(
t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")]
)
self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.")

# Test disallowed mix of t-string and string/f-string (incl. bytes)
what = 't'
expected_msg = 'cannot mix bytes and nonbytes literals'
expected_msg = 'cannot mix t-strings with strings or f-strings'
for case in (
"t'{what}-string literal' 'str literal'",
"t'{what}-string literal' u'unicode literal'",
"t'{what}-string literal' f'f-string literal'",
"t'{what}-string literal' r'raw string literal'",
"t'{what}-string literal' rf'raw f-string literal'",
"t'{what}-string literal' b'bytes literal'",
"t'{what}-string literal' br'raw bytes literal'",
"'str literal' t'{what}-string literal'",
"u'unicode literal' t'{what}-string literal'",
"f'f-string literal' t'{what}-string literal'",
"r'raw string literal' t'{what}-string literal'",
"rf'raw f-string literal' t'{what}-string literal'",
"b'bytes literal' t'{what}-string literal'",
"br'raw bytes literal' t'{what}-string literal'",
):
with self.assertRaisesRegex(SyntaxError, expected_msg):
eval(case)
with self.subTest(case):
with self.assertRaisesRegex(SyntaxError, expected_msg):
eval(case)

def test_triple_quoted(self):
# Test triple-quoted t-strings
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_unparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,6 @@ def test_tstrings(self):
self.check_ast_roundtrip("t'foo'")
self.check_ast_roundtrip("t'foo {bar}'")
self.check_ast_roundtrip("t'foo {bar!s:.2f}'")
self.check_ast_roundtrip("t'foo {bar}' f'{bar}'")
self.check_ast_roundtrip("f'{bar}' t'foo {bar}'")
self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'")
self.check_ast_roundtrip("t'foo {bar}' u'bar'")

def test_strings(self):
self.check_ast_roundtrip("u'foo'")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Reflect recent :pep:`750` change.

Disallow concatenation of :class:`string.Template` and :class:`str`. Also, disallow implicit
concatenation of t-string literals with string or f-string literals.
91 changes: 7 additions & 84 deletions Objects/templateobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ templateiter_next(PyObject *op)
Py_SETREF(item, PyIter_Next(self->interpolationsiter));
self->from_strings = 1;
}
} else {
}
else {
item = PyIter_Next(self->interpolationsiter);
self->from_strings = 1;
}
Expand Down Expand Up @@ -245,54 +246,6 @@ template_iter(PyObject *op)
return (PyObject *)iter;
}

static PyObject *
template_strings_append_str(PyObject *strings, PyObject *str)
{
Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1);
PyObject *concat = PyUnicode_Concat(string, str);
if (concat == NULL) {
return NULL;
}

PyObject *newstrings = PyTuple_New(stringslen);
if (newstrings == NULL) {
Py_DECREF(concat);
return NULL;
}

for (Py_ssize_t i = 0; i < stringslen - 1; i++) {
PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
}
PyTuple_SET_ITEM(newstrings, stringslen - 1, concat);

return newstrings;
}

static PyObject *
template_strings_prepend_str(PyObject *strings, PyObject *str)
{
Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
PyObject *string = PyTuple_GET_ITEM(strings, 0);
PyObject *concat = PyUnicode_Concat(str, string);
if (concat == NULL) {
return NULL;
}

PyObject *newstrings = PyTuple_New(stringslen);
if (newstrings == NULL) {
Py_DECREF(concat);
return NULL;
}

PyTuple_SET_ITEM(newstrings, 0, concat);
for (Py_ssize_t i = 1; i < stringslen; i++) {
PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
}

return newstrings;
}

static PyObject *
template_strings_concat(PyObject *left, PyObject *right)
{
Expand Down Expand Up @@ -344,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other)
return newtemplate;
}

static PyObject *
template_concat_template_str(templateobject *self, PyObject *other)
{
PyObject *newstrings = template_strings_append_str(self->strings, other);
if (newstrings == NULL) {
return NULL;
}

PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
Py_DECREF(newstrings);
return newtemplate;
}

static PyObject *
template_concat_str_template(templateobject *self, PyObject *other)
{
PyObject *newstrings = template_strings_prepend_str(self->strings, other);
if (newstrings == NULL) {
return NULL;
}

PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
Py_DECREF(newstrings);
return newtemplate;
}

PyObject *
_PyTemplate_Concat(PyObject *self, PyObject *other)
{
if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) {
return template_concat_templates((templateobject *) self, (templateobject *) other);
}
else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) {
return template_concat_template_str((templateobject *) self, other);
}
else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) {
return template_concat_str_template((templateobject *) other, self);
}
else {
Py_RETURN_NOTIMPLEMENTED;
}

PyErr_Format(PyExc_TypeError,
"can only concatenate Template (not \"%T\") to Template",
other);
return NULL;
}

static PyObject *
Expand Down
15 changes: 4 additions & 11 deletions Objects/unicodeobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#include "pycore_pyhash.h" // _Py_HashSecret_t
#include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_template.h" // _PyTemplate_Concat()
#include "pycore_tuple.h" // _PyTuple_FromArray()
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
#include "pycore_unicodeobject.h" // struct _Py_unicode_state
Expand Down Expand Up @@ -11610,16 +11609,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
return NULL;

if (!PyUnicode_Check(right)) {
if (_PyTemplate_CheckExact(right)) {
// str + tstring is implemented in the tstring type
return _PyTemplate_Concat(left, right);
}
else {
PyErr_Format(PyExc_TypeError,
"can only concatenate str (not \"%.200s\") to str",
Py_TYPE(right)->tp_name);
return NULL;
}
PyErr_Format(PyExc_TypeError,
"can only concatenate str (not \"%.200s\") to str",
Py_TYPE(right)->tp_name);
return NULL;
}

/* Shortcuts */
Expand Down
17 changes: 4 additions & 13 deletions Parser/action_helpers.c
Original file line number Diff line number Diff line change
Expand Up @@ -1834,8 +1834,8 @@ _build_concatenated_joined_str(Parser *p, asdl_expr_seq *strings,
return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena);
}

static expr_ty
_build_concatenated_template_str(Parser *p, asdl_expr_seq *strings,
expr_ty
_PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *strings,
int lineno, int col_offset, int end_lineno,
int end_col_offset, PyArena *arena)
{
Expand All @@ -1853,7 +1853,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
Py_ssize_t len = asdl_seq_LEN(strings);
assert(len > 0);

int t_string_found = 0;
int f_string_found = 0;
int unicode_string_found = 0;
int bytes_found = 0;
Expand All @@ -1872,23 +1871,20 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
case JoinedStr_kind:
f_string_found = 1;
break;
case TemplateStr_kind:
t_string_found = 1;
break;
default:
f_string_found = 1;
break;
}
}

// Cannot mix unicode and bytes
if ((unicode_string_found || f_string_found || t_string_found) && bytes_found) {
if ((unicode_string_found || f_string_found) && bytes_found) {
RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
return NULL;
}

// If it's only bytes or only unicode string, do a simple concat
if (!f_string_found && !t_string_found) {
if (!f_string_found) {
if (len == 1) {
return asdl_seq_GET(strings, 0);
}
Expand All @@ -1902,11 +1898,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
}
}

if (t_string_found) {
return _build_concatenated_template_str(p, strings, lineno,
col_offset, end_lineno, end_col_offset, arena);
}

return _build_concatenated_joined_str(p, strings, lineno,
col_offset, end_lineno, end_col_offset, arena);
}
Expand Down
Loading
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy