Skip to content

Commit 7b88fd4

Browse files
gwostigwosti
committed
gh-155525: Avoid quadratic f-string tokenization
1 parent eb087d4 commit 7b88fd4

10 files changed

Lines changed: 76 additions & 118 deletions

File tree

‎Lib/test/test_fstring.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -832,6 +832,14 @@ def build_fstr(n, extra=''):
832832
s = "f'{1}' 'x' 'y'" * 1024
833833
self.assertEqual(eval(s), '1xy' * 1024)
834834

835+
@support.requires_resource('cpu')
836+
def test_many_fstrings_in_module(self):
837+
fields = ''.join(f'{{x{i}}}' for i in range(100))
838+
source = ''.join(
839+
f"value_{i} = f'{fields}'\n" for i in range(1_000)
840+
)
841+
compile(source, '<string>', 'exec')
842+
835843
def test_format_specifier_expressions(self):
836844
width = 10
837845
precision = 4
@@ -1796,6 +1804,19 @@ def test_debug_in_file(self):
17961804
self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'),
17971805
"3\n=3")
17981806

1807+
def test_debug_in_file_after_buffer_resize(self):
1808+
expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)"
1809+
expected = expression + "=1"
1810+
with temp_cwd():
1811+
script = 'script.py'
1812+
source = (
1813+
f"result = f'''{{{expression}=}}'''\n"
1814+
f"assert result == {expected!r}\n"
1815+
)
1816+
with open(script, 'w') as f:
1817+
f.write(source)
1818+
assert_python_ok(script)
1819+
17991820
def test_syntax_warning_infinite_recursion_in_file(self):
18001821
with temp_cwd():
18011822
script = 'script.py'

‎Lib/test/test_tstring.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import unittest
22

3+
from test import support
4+
from test.support.os_helper import temp_cwd
5+
from test.support.script_helper import assert_python_ok
36
from test.test_string._support import TStringBaseCase, fstring
47

58

@@ -79,6 +82,27 @@ def upper(self):
7982
)
8083
self.assertEqual(fstring(t), "Name: Bob, Age: 30")
8184

85+
def test_interpolation_expression_in_file_after_buffer_resize(self):
86+
expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)"
87+
with temp_cwd():
88+
script = 'script.py'
89+
source = (
90+
f"template = t'''{{{expression}}}'''\n"
91+
"interpolation = template.interpolations[0]\n"
92+
f"assert interpolation.expression == {expression!r}\n"
93+
)
94+
with open(script, 'w') as f:
95+
f.write(source)
96+
assert_python_ok(script)
97+
98+
@support.requires_resource('cpu')
99+
def test_many_tstrings_in_module(self):
100+
fields = ''.join(f'{{x{i}}}' for i in range(100))
101+
source = ''.join(
102+
f"value_{i} = t'{fields}'\n" for i in range(1_000)
103+
)
104+
compile(source, '<string>', 'exec')
105+
82106
def test_format_specifiers(self):
83107
# Test basic format specifiers
84108
value = 3.14159

‎Misc/NEWS.d/next/Core_and_Builtins/2026-09-01-07-15-20.gh-issue-155525.A7kP2m.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix quadratic-time tokenization of modules containing many f-strings or
2+
t-strings.

‎Parser/lexer/buffer.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ _PyLexer_remember_fstring_buffers(struct tok_state *tok)
1515
mode = &(tok->tok_mode_stack[index]);
1616
mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
1717
mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
18+
mode->last_expr_start_offset = mode->last_expr_start == NULL
19+
? -1 : mode->last_expr_start - tok->buf;
1820
}
1921
}
2022

@@ -29,6 +31,8 @@ _PyLexer_restore_fstring_buffers(struct tok_state *tok)
2931
mode = &(tok->tok_mode_stack[index]);
3032
mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
3133
mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
34+
mode->last_expr_start = mode->last_expr_start_offset < 0
35+
? NULL : tok->buf + mode->last_expr_start_offset;
3236
}
3337
}
3438

‎Parser/lexer/lexer.c‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -542,9 +542,6 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
542542
int cursor_in_format_with_debug =
543543
cursor == 1 && (current_tok->in_debug || in_format_spec);
544544
int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
545-
if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
546-
return MAKE_TOKEN(ENDMARKER);
547-
}
548545
if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
549546
return MAKE_TOKEN(ERRORTOKEN);
550547
}

‎Parser/lexer/lexer.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33

44
#include "state.h"
55

6-
int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
7-
86
int _PyTokenizer_Get(struct tok_state *, struct token *);
97

108
#endif

‎Parser/lexer/state.c‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -61,24 +61,6 @@ _PyTokenizer_tok_new(void)
6161
return tok;
6262
}
6363

64-
static void
65-
free_fstring_expressions(struct tok_state *tok)
66-
{
67-
int index;
68-
tokenizer_mode *mode;
69-
70-
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
71-
mode = &(tok->tok_mode_stack[index]);
72-
if (mode->last_expr_buffer != NULL) {
73-
PyMem_Free(mode->last_expr_buffer);
74-
mode->last_expr_buffer = NULL;
75-
mode->last_expr_size = 0;
76-
mode->last_expr_end = -1;
77-
mode->in_format_spec = 0;
78-
}
79-
}
80-
}
81-
8264
/* Free a tok_state structure */
8365
void
8466
_PyTokenizer_Free(struct tok_state *tok)
@@ -90,7 +72,6 @@ _PyTokenizer_Free(struct tok_state *tok)
9072
Py_XDECREF(tok->module);
9173
_PyTok_ReaderFree(tok);
9274
_PyTok_SourceClear(&tok->source);
93-
free_fstring_expressions(tok);
9475
PyMem_Free(tok);
9576
}
9677

‎Parser/lexer/state.h‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,8 @@ typedef struct _tokenizer_mode {
5656
Py_ssize_t start_offset;
5757
Py_ssize_t multi_line_start_offset;
5858

59-
Py_ssize_t last_expr_size;
60-
Py_ssize_t last_expr_end;
61-
char* last_expr_buffer;
59+
const char* last_expr_start;
60+
Py_ssize_t last_expr_start_offset;
6261
int in_debug;
6362
int in_format_spec;
6463

‎Parser/lexer/string.c‎

Lines changed: 23 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,19 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
1616
if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
1717
return 0;
1818
}
19+
const char *expression = tok_mode->last_expr_start;
20+
assert(expression != NULL);
21+
assert(expression <= tok->start);
22+
Py_ssize_t expression_size = tok->start - expression;
1923
PyObject *res = NULL;
2024

2125
// Look for a # character outside of string literals
2226
int hash_detected = 0;
2327
int in_string = 0;
2428
char quote_char = 0;
2529

26-
for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
27-
char ch = tok_mode->last_expr_buffer[i];
30+
for (Py_ssize_t i = 0; i < expression_size; i++) {
31+
char ch = expression[i];
2832

2933
// Skip escaped characters
3034
if (ch == '\\') {
@@ -60,7 +64,8 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
6064
// If we found a # character in the expression, we need to handle comments
6165
if (hash_detected) {
6266
// Allocate buffer for processed result
63-
char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
67+
char *result = (char *)PyMem_Malloc(
68+
(expression_size + 1) * sizeof(char));
6469
if (!result) {
6570
return -1;
6671
}
@@ -71,8 +76,8 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
7176
quote_char = 0; // Current string quote char
7277

7378
// Process each character
74-
while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
75-
char ch = tok_mode->last_expr_buffer[i];
79+
while (i < expression_size) {
80+
char ch = expression[i];
7681

7782
// Handle string quotes
7883
if (ch == '"' || ch == '\'') {
@@ -87,11 +92,10 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
8792
}
8893
// Skip comments
8994
else if (ch == '#' && !in_string) {
90-
while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
91-
tok_mode->last_expr_buffer[i] != '\n') {
95+
while (i < expression_size && expression[i] != '\n') {
9296
i++;
9397
}
94-
if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
98+
if (i < expression_size) {
9599
result[j++] = '\n';
96100
}
97101
}
@@ -106,11 +110,7 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
106110
res = PyUnicode_DecodeUTF8(result, j, NULL);
107111
PyMem_Free(result);
108112
} else {
109-
res = PyUnicode_DecodeUTF8(
110-
tok_mode->last_expr_buffer,
111-
tok_mode->last_expr_size - tok_mode->last_expr_end,
112-
NULL
113-
);
113+
res = PyUnicode_DecodeUTF8(expression, expression_size, NULL);
114114
}
115115

116116
if (!res) {
@@ -120,61 +120,6 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
120120
return 0;
121121
}
122122

123-
int
124-
_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
125-
{
126-
assert(tok->cur != NULL);
127-
128-
Py_ssize_t size = strlen(tok->cur);
129-
tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
130-
131-
switch (cur) {
132-
case 0:
133-
if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
134-
return 1;
135-
}
136-
char *new_buffer = PyMem_Realloc(
137-
tok_mode->last_expr_buffer,
138-
tok_mode->last_expr_size + size
139-
);
140-
if (new_buffer == NULL) {
141-
PyMem_Free(tok_mode->last_expr_buffer);
142-
goto error;
143-
}
144-
tok_mode->last_expr_buffer = new_buffer;
145-
strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
146-
tok_mode->last_expr_size += size;
147-
break;
148-
case '{':
149-
if (tok_mode->last_expr_buffer != NULL) {
150-
PyMem_Free(tok_mode->last_expr_buffer);
151-
}
152-
tok_mode->last_expr_buffer = PyMem_Malloc(size);
153-
if (tok_mode->last_expr_buffer == NULL) {
154-
goto error;
155-
}
156-
tok_mode->last_expr_size = size;
157-
tok_mode->last_expr_end = -1;
158-
strncpy(tok_mode->last_expr_buffer, tok->cur, size);
159-
break;
160-
case '}':
161-
case '!':
162-
tok_mode->last_expr_end = strlen(tok->start);
163-
break;
164-
case ':':
165-
if (tok_mode->last_expr_end == -1) {
166-
tok_mode->last_expr_end = strlen(tok->start);
167-
}
168-
break;
169-
default:
170-
Py_UNREACHABLE();
171-
}
172-
return 1;
173-
error:
174-
tok->done = E_NOMEM;
175-
return 0;
176-
}
177-
178123
int
179124
_PyLexer_check_string_prefixes(struct tok_state *tok,
180125
int saw_b, int saw_r, int saw_u,
@@ -268,9 +213,8 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
268213
the_current_tok->first_line = tok->lineno;
269214
the_current_tok->start_offset = -1;
270215
the_current_tok->multi_line_start_offset = -1;
271-
the_current_tok->last_expr_buffer = NULL;
272-
the_current_tok->last_expr_size = 0;
273-
the_current_tok->last_expr_end = -1;
216+
the_current_tok->last_expr_start = NULL;
217+
the_current_tok->last_expr_start_offset = -1;
274218
the_current_tok->in_format_spec = 0;
275219
the_current_tok->in_debug = 0;
276220

@@ -436,6 +380,9 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
436380
if (start_char == '{') {
437381
int peek1 = tok_nextc(tok);
438382
tok_backup(tok, peek1);
383+
if (peek1 != '{') {
384+
current_tok->last_expr_start = tok->cur;
385+
}
439386
tok_backup(tok, start_char);
440387
if (peek1 != '{') {
441388
current_tok->curly_bracket_expr_start_depth++;
@@ -460,13 +407,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
460407
}
461408
}
462409

463-
if (current_tok->last_expr_buffer != NULL) {
464-
PyMem_Free(current_tok->last_expr_buffer);
465-
current_tok->last_expr_buffer = NULL;
466-
current_tok->last_expr_size = 0;
467-
current_tok->last_expr_end = -1;
468-
}
469-
470410
p_start = tok->start;
471411
p_end = tok->cur;
472412
tok->tok_mode_stack_index--;
@@ -551,12 +491,10 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
551491
}
552492

553493
if (c == '{') {
554-
if (!_PyLexer_update_ftstring_expr(tok, c)) {
555-
return MAKE_TOKEN(ENDMARKER);
556-
}
557494
int peek = tok_nextc(tok);
558495
if (peek != '{' || in_format_spec) {
559496
tok_backup(tok, peek);
497+
current_tok->last_expr_start = tok->cur;
560498
tok_backup(tok, c);
561499
current_tok->curly_bracket_expr_start_depth++;
562500
if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
@@ -580,10 +518,10 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
580518
}
581519
int peek = tok_nextc(tok);
582520

583-
// The tokenizer can only be in the format spec if we have already completed the expression
584-
// scanning (indicated by the end of the expression being set) and we are not at the top level
585-
// of the bracket stack (-1 is the top level). Since format specifiers can't legally use double
586-
// brackets, we can bypass it here.
521+
// The tokenizer can only be in the format spec if expression
522+
// scanning is complete and we are not at the top level of the
523+
// bracket stack (-1 is the top level). Since format specifiers
524+
// can't legally use double brackets, we can bypass it here.
587525
int cursor = current_tok->curly_bracket_depth;
588526
if (peek == '}' && !in_format_spec && cursor == 0) {
589527
p_start = tok->start;

‎Parser/tokenizer/reader.c‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -604,12 +604,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
604604
}
605605
tok->implicit_newline = chunk.implicit_newline;
606606

607-
if (!prepared && tok->tok_mode_stack_index &&
608-
!_PyLexer_update_ftstring_expr(tok, 0)) {
609-
_PyTok_ChunkClear(&chunk);
610-
tok->input_error = 1;
611-
return 0;
612-
}
613607
ADVANCE_LINENO();
614608
if (tok->reader->kind == _PYTOK_READER_FILE &&
615609
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&

0 commit comments

Comments
 (0)