From c85f33004dd4d14f405bb92bca4b9bdc54a93302 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 15 Sep 2026 22:23:59 +0300 Subject: [PATCH 1/4] gh-69919: Raise SyntaxError for all invalid sources in compile() compile(), exec(), eval() and ast.parse() now raise SyntaxError instead of ValueError if the source string contains surrogate characters, and instead of MemoryError or RecursionError if the source is too complex to parse or compile. Consumers that handle SyntaxError, like the code module and IDLE, no longer need to handle those exceptions. Co-Authored-By: Claude Opus 5 (1M context) --- Doc/builtins/functions.rst | 22 +++++- Include/internal/pycore_compile.h | 2 + Include/internal/pycore_pythonrun.h | 1 + Lib/test/test_ast/test_ast.py | 3 +- Lib/test/test_builtin.py | 7 ++ Lib/test/test_code_module.py | 12 ++- Lib/test/test_compile.py | 3 +- Lib/test/test_syntax.py | 2 +- ...6-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst | 4 + Modules/symtablemodule.c | 2 +- Parser/pegen.c | 2 + Parser/pegen_errors.c | 3 +- Python/bltinmodule.c | 8 +- Python/compile.c | 17 +++++ Python/crossinterp.c | 2 +- Python/pythonrun.c | 76 ++++++++++++++++--- 16 files changed, 137 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index 5cce5e3c87628a..b70ffb08d0371e 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -384,8 +384,12 @@ are always available. They are listed here in alphabetical order. It is needed to unambiguous :ref:`filter ` syntax warnings by module name. - This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled - source is invalid. + This function raises :exc:`SyntaxError` if the compiled source is invalid, + including a *source* containing a null or surrogate character, + that cannot be decoded, + or that is too complex to parse or compile, + for example an expression with many thousands of nested operators; + and :exc:`ValueError` if *mode* or *flags* is invalid. If you want to parse Python code into its AST representation, see :func:`ast.parse`. @@ -417,13 +421,23 @@ are always available. They are listed here in alphabetical order. Previously, :exc:`TypeError` was raised when null bytes were encountered in *source*. - .. versionadded:: 3.8 + .. versionchanged:: 3.8 ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. - .. versionadded:: 3.15 + .. versionchanged:: 3.12 + :exc:`SyntaxError` is raised instead of :exc:`ValueError` when null bytes + are encountered in *source*. + + .. versionchanged:: 3.15 Added the *module* parameter. + .. versionchanged:: 3.16 + :exc:`SyntaxError` is raised instead of :exc:`ValueError` when surrogate + characters are encountered in *source*, + and instead of :exc:`MemoryError` or :exc:`RecursionError` + when *source* is too complex. + .. class:: complex(number=0, /) complex(string, /) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 4597ae2763ad77..7d466461841b49 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -36,6 +36,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyObject *module); /* AST preprocessing */ +extern void _PyCompile_CheckRecursionError(void); + extern int _PyCompile_AstPreprocess( struct _mod *mod, PyObject *filename, diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index d333eb2ccf7c41..4743436ed77cb5 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -25,6 +25,7 @@ extern const char* _Py_SourceAsString( PyObject *cmd, const char *funcname, const char *what, + PyObject *filename, PyCompilerFlags *cf, PyObject **cmd_copy); diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index bb282eba5d41ee..fa79bffee0d299 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1126,7 +1126,8 @@ def check_limit(prefix, repeated): broken = prefix + repeated * crash_depth details = "Compiling ({!r} + {!r} * {})".format( prefix, repeated, crash_depth) - with self.assertRaises(RecursionError, msg=details): + with self.assertRaisesRegex(SyntaxError, "too complex to compile", + msg=details): with support.infinite_recursion(): ast.parse(broken) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 294355698c8683..4a9a1a18676384 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -446,6 +446,13 @@ def test_compile(self): mode='eval', source='0', filename='tmp') compile('print("\xe5")\n', '', 'exec') self.assertRaises(SyntaxError, compile, chr(0), 'f', 'exec') + with self.assertRaises(SyntaxError) as cm: + compile("x = 1\ny = 'ab\udc80cd'\n", 'f', 'exec') + self.assertEqual((cm.exception.filename, cm.exception.lineno, + cm.exception.offset, cm.exception.text), + ('f', 2, 8, "y = 'ab\udc80cd'")) + self.assertRaises(SyntaxError, eval, '\udc80') + self.assertRaises(SyntaxError, exec, '\udc80') self.assertRaises(ValueError, compile, str('a = 1'), 'f', 'bad') # test the optimize argument diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 3642b47c2c1f03..f90589ecc7a863 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -132,10 +132,14 @@ def test_unicode_error(self): self.console.interact() output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) output = output[output.index('(InteractiveConsole)'):] - output = output[output.index('\n') + 1:] - self.assertStartsWith(output, 'UnicodeEncodeError: ') - self.assertIs(self.sysmod.last_type, UnicodeEncodeError) - self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 1', + " '\ud800'", + ' ^', + 'SyntaxError: source code string cannot contain surrogate characters']) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) self.assertIsNone(self.sysmod.last_traceback) self.assertIsNone(self.sysmod.last_value.__traceback__) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 959732fc6e4a83..484a927f768415 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -763,7 +763,8 @@ def check_limit(prefix, repeated, mode="single"): compile(expect_ok, '', mode) broken = prefix + repeated * crash_depth details = f"Compiling ({prefix!r} + {repeated!r} * {crash_depth})" - with self.assertRaises(RecursionError, msg=details): + with self.assertRaisesRegex(SyntaxError, "too complex to compile", + msg=details): compile(broken, '', mode) check_limit("a", "()") diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 5358b6f5ee8fcd..7063b71ecdb15d 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -3543,7 +3543,7 @@ def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" for mode in ["exec", "eval", "single"]: with self.subTest(mode=mode): - with self.assertRaisesRegex(MemoryError, r"too complex"): + with self.assertRaisesRegex(SyntaxError, r"too complex to parse"): compile(source, "", mode) @support.cpython_only diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst new file mode 100644 index 00000000000000..72ac6f630b3d85 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst @@ -0,0 +1,4 @@ +:func:`compile`, :func:`exec`, :func:`eval` and :func:`ast.parse` now raise +:exc:`SyntaxError` instead of :exc:`ValueError` if the source string contains +surrogate characters, and instead of :exc:`MemoryError` or +:exc:`RecursionError` if the source is too complex to parse or compile. diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 7e20b5c7173ae5..2aacd36f9f680c 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -92,7 +92,7 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, cf.cf_flags = PyCF_SOURCE_IS_UTF8; const char *str = _Py_SourceAsString(source, "symtable", "string, bytes or AST", - &cf, &source_copy); + filename, &cf, &source_copy); if (str == NULL) { return NULL; } diff --git a/Parser/pegen.c b/Parser/pegen.c index b13cb8d400f9c3..46f2b9b304c91b 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -2,6 +2,7 @@ #include "pycore_ast.h" // _PyAST_Validate(), #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_parser.h" // _PYPEGEN_NSTATISTICS +#include "pycore_compile.h" // _PyCompile_CheckRecursionError() #include "pycore_pyerrors.h" // PyExc_IncompleteInputError #include "pycore_runtime.h" // _PyRuntime #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal @@ -1063,6 +1064,7 @@ _PyPegen_run_parser(Parser *p) p->start_rule == Py_eval_input) { if (!_PyAST_Validate(res)) { + _PyCompile_CheckRecursionError(); return NULL; } } diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 7841f01b612915..1c85d981ebafde 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -391,6 +391,5 @@ void _Pypegen_stack_overflow(Parser *p) { p->error_indicator = 1; - PyErr_SetString(PyExc_MemoryError, - "Parser stack overflowed - Python source too complex to parse"); + RAISE_SYNTAX_ERROR("Python source too complex to parse"); } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d28e6fa9cd01ae..0e834a41647ab9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -949,7 +949,8 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto finally; } - str = _Py_SourceAsString(source, "compile", "string, bytes or AST", &cf, &source_copy); + str = _Py_SourceAsString(source, "compile", "string, bytes or AST", + filename, &cf, &source_copy); if (str == NULL) goto error; @@ -1116,7 +1117,8 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, else { PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags = PyCF_SOURCE_IS_UTF8; - str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy); + str = _Py_SourceAsString(source, "eval", "string, bytes or code", + NULL, &cf, &source_copy); if (str == NULL) goto error; @@ -1286,7 +1288,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags = PyCF_SOURCE_IS_UTF8; str = _Py_SourceAsString(source, "exec", - "string, bytes or code", &cf, + "string, bytes or code", NULL, &cf, &source_copy); if (str == NULL) goto error; diff --git a/Python/compile.c b/Python/compile.c index f3852041bce69c..e0ecff017189c2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1524,6 +1524,18 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) return optimize_and_assemble_code_unit(u, const_cache, code_flags, filename); } +/* Replace a RecursionError raised while processing too deeply nested + source with a SyntaxError, as the parser raises for such source. */ +void +_PyCompile_CheckRecursionError(void) +{ + if (PyErr_ExceptionMatches(PyExc_RecursionError)) { + PyErr_Clear(); + PyErr_SetString(PyExc_SyntaxError, + "Python source too complex to compile"); + } +} + PyCodeObject * _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int optimize, PyArena *arena, PyObject *module) @@ -1531,12 +1543,16 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, assert(!PyErr_Occurred()); compiler *c = new_compiler(mod, filename, pflags, optimize, arena, module); if (c == NULL) { + _PyCompile_CheckRecursionError(); return NULL; } PyCodeObject *co = compiler_mod(c, mod); compiler_free(c); assert(co || PyErr_Occurred()); + if (co == NULL) { + _PyCompile_CheckRecursionError(); + } return co; } @@ -1556,6 +1572,7 @@ _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, no_const_folding, 0, module)) { + _PyCompile_CheckRecursionError(); return -1; } return 0; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index ed77c1be646e27..aa7b6359b00f0a 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -943,7 +943,7 @@ get_script_xidata(PyThreadState *tstate, PyObject *obj, int pure, PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags = PyCF_SOURCE_IS_UTF8; PyObject *ref = NULL; - const char *script = _Py_SourceAsString(obj, "???", "???", &cf, &ref); + const char *script = _Py_SourceAsString(obj, "???", "???", NULL, &cf, &ref); if (script == NULL) { if (!_PyObject_SupportedAsScript(obj)) { // We discard the raised exception. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 049eaea4994ac4..f39c32201a5cba 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1575,26 +1575,30 @@ _Py_CompileString(const char *str, PyObject *filename, int start, if (arena == NULL) return NULL; + PyObject *result = NULL; mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { - _PyArena_Free(arena); - return NULL; + goto done; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, syntax_check_only, module) < 0) { - _PyArena_Free(arena); - return NULL; + goto done; } - PyObject *result = PyAST_mod2obj(mod); - _PyArena_Free(arena); - return result; + result = PyAST_mod2obj(mod); + } + else { + co = _PyAST_Compile(mod, filename, flags, optimize, arena, module); + result = (PyObject *)co; } - co = _PyAST_Compile(mod, filename, flags, optimize, arena, module); +done: _PyArena_Free(arena); - return (PyObject *)co; + if (result == NULL) { + _PyCompile_CheckRecursionError(); + } + return result; } PyObject * @@ -1630,8 +1634,48 @@ _PyObject_SupportedAsScript(PyObject *cmd) } } +/* Raise a SyntaxError for the character at index pos of the source string. */ +static void +source_syntax_error(PyObject *source, PyObject *filename, Py_ssize_t pos, + const char *msg) +{ + Py_ssize_t len = PyUnicode_GET_LENGTH(source); + Py_ssize_t lineno = 1; + Py_ssize_t linestart = 0; + for (Py_ssize_t i = 0; i < pos; i++) { + if (PyUnicode_READ_CHAR(source, i) == '\n') { + lineno++; + linestart = i + 1; + } + } + Py_ssize_t lineend = PyUnicode_FindChar(source, '\n', pos, len, 1); + if (lineend < 0) { + lineend = len; + } + PyObject *text = PyUnicode_Substring(source, linestart, lineend); + if (text == NULL) { + return; + } + Py_ssize_t offset = pos - linestart + 1; + PyObject *args; + if (filename != NULL) { + args = Py_BuildValue("s(OnnNnn)", msg, filename, + lineno, offset, text, lineno, offset + 1); + } + else { + args = Py_BuildValue("s(snnNnn)", msg, "", + lineno, offset, text, lineno, offset + 1); + } + if (args == NULL) { + return; + } + PyErr_SetObject(PyExc_SyntaxError, args); + Py_DECREF(args); +} + const char * -_Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy) +_Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, + PyObject *filename, PyCompilerFlags *cf, PyObject **cmd_copy) { const char *str; Py_ssize_t size; @@ -1641,8 +1685,18 @@ _Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyComp if (PyUnicode_Check(cmd)) { cf->cf_flags |= PyCF_IGNORE_COOKIE; str = PyUnicode_AsUTF8AndSize(cmd, &size); - if (str == NULL) + if (str == NULL) { + if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { + PyObject *exc = PyErr_GetRaisedException(); + Py_ssize_t pos; + if (PyUnicodeEncodeError_GetStart(exc, &pos) == 0) { + source_syntax_error(cmd, filename, pos, + "source code string cannot contain surrogate characters"); + } + Py_DECREF(exc); + } return NULL; + } } else if (PyBytes_Check(cmd)) { str = PyBytes_AS_STRING(cmd); From 7d45307341ba008af48bd2344b9d801380e85704 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 15 Sep 2026 22:38:20 +0300 Subject: [PATCH 2/4] Use 'next' as the version in versionchanged --- Doc/builtins/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index b70ffb08d0371e..0853952fa4a00c 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -432,7 +432,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.15 Added the *module* parameter. - .. versionchanged:: 3.16 + .. versionchanged:: next :exc:`SyntaxError` is raised instead of :exc:`ValueError` when surrogate characters are encountered in *source*, and instead of :exc:`MemoryError` or :exc:`RecursionError` From 76e57412b0d35cae4d1c20fb23b70b0f50ae6123 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 15 Sep 2026 22:57:30 +0300 Subject: [PATCH 3/4] Raise SyntaxError instead of OverflowError for too large sources A line longer than 2**31 bytes, more than 2**31 lines, or a string literal longer than 2**31 bytes are syntax errors too. Drop the stale mentions of OverflowError and ValueError for invalid literals in the code and codeop documentation. --- Doc/builtins/functions.rst | 10 ++++++---- Doc/library/ast.rst | 5 ++--- Doc/library/code.rst | 8 +++----- Doc/library/codeop.rst | 4 +--- Lib/code.py | 2 +- Lib/codeop.py | 12 +++--------- .../2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst | 5 +++-- Parser/pegen_errors.c | 5 ++--- Parser/string_parser.c | 2 +- Parser/tokenizer/source.c | 2 +- 10 files changed, 23 insertions(+), 32 deletions(-) diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index 0853952fa4a00c..6ffeb12b415426 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -387,8 +387,9 @@ are always available. They are listed here in alphabetical order. This function raises :exc:`SyntaxError` if the compiled source is invalid, including a *source* containing a null or surrogate character, that cannot be decoded, - or that is too complex to parse or compile, - for example an expression with many thousands of nested operators; + that is too complex to parse or compile, + for example an expression with many thousands of nested operators, + or that is too large; and :exc:`ValueError` if *mode* or *flags* is invalid. If you want to parse Python code into its AST representation, see @@ -435,8 +436,9 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: next :exc:`SyntaxError` is raised instead of :exc:`ValueError` when surrogate characters are encountered in *source*, - and instead of :exc:`MemoryError` or :exc:`RecursionError` - when *source* is too complex. + instead of :exc:`MemoryError` or :exc:`RecursionError` + when *source* is too complex, + and instead of :exc:`OverflowError` when *source* is too large. .. class:: complex(number=0, /) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index e30dadc9733d37..424de8044e616f 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2338,9 +2338,8 @@ and classes for traversing abstract syntax trees: It is possible to crash the Python interpreter due to stack depth limitations in Python's AST compiler. - It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, - :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed - input. + It can raise :exc:`ValueError`, :exc:`TypeError` or :exc:`SyntaxError` + depending on the malformed input. .. versionchanged:: 3.2 Now allows bytes and set literals. diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 59c016d21501b0..70a81b178f26b9 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -74,9 +74,7 @@ build applications which provide an interactive interpreter prompt. Returns a code object (the same as ``compile(source, filename, symbol)``) if the command is complete and valid; ``None`` if the command is incomplete; raises - :exc:`SyntaxError` if the command is complete and contains a syntax error, or - raises :exc:`OverflowError` or :exc:`ValueError` if the command contains an - invalid literal. + :exc:`SyntaxError` if the command is complete and invalid. .. _interpreter-objects: @@ -91,8 +89,8 @@ Interactive Interpreter Objects :func:`compile_command`; the default for *filename* is ``''``, and for *symbol* is ``'single'``. One of several things can happen: - * The input is incorrect; :func:`compile_command` raised an exception - (:exc:`SyntaxError` or :exc:`OverflowError`). A syntax traceback will be + * The input is incorrect; :func:`compile_command` raised + :exc:`SyntaxError`. A syntax traceback will be printed by calling the :meth:`showsyntaxerror` method. :meth:`runsource` returns ``False``. diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 622e57d2ee63db..ac50a0e2d981e8 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -35,9 +35,7 @@ To do just the former: ``''``. Returns ``None`` if *source* is *not* valid Python code, but is a prefix of valid Python code. - If there is a problem with *source*, an exception will be raised. - :exc:`SyntaxError` is raised if there is invalid Python syntax, and - :exc:`OverflowError` or :exc:`ValueError` if there is an invalid literal. + If there is a problem with *source*, :exc:`SyntaxError` is raised. The *symbol* argument determines whether *source* is compiled as a statement (``'single'``, the default), as a sequence of :term:`statement` (``'exec'``) or diff --git a/Lib/code.py b/Lib/code.py index df1d7199e33934..28509155637d39 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -44,7 +44,7 @@ def runsource(self, source, filename="", symbol="single"): One of several things can happen: 1) The input is incorrect; compile_command() raised an - exception (SyntaxError or OverflowError). A syntax traceback + exception (SyntaxError). A syntax traceback will be printed by calling the showsyntaxerror() method. 2) The input is incomplete, and more input is required; diff --git a/Lib/codeop.py b/Lib/codeop.py index 40e88423119bc4..a314029b42870e 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -6,9 +6,7 @@ - Return code object if the command is complete and valid - Return None if the command is incomplete -- Raise SyntaxError, ValueError or OverflowError if the command is a - syntax error (OverflowError and ValueError can be produced by - malformed literals). +- Raise SyntaxError if the command is a syntax error. The two interfaces are: @@ -95,9 +93,7 @@ def compile_command(source, filename="", symbol="single", flags=0): - Return a code object if the command is complete and valid - Return None if the command is incomplete - - Raise SyntaxError, ValueError or OverflowError if the command is a - syntax error (OverflowError and ValueError can be produced by - malformed literals). + - Raise SyntaxError if the command is a syntax error. """ return _maybe_compile(_compile, source, filename, symbol, flags) @@ -147,8 +143,6 @@ def __call__(self, source, filename="", symbol="single"): - Return a code object if the command is complete and valid - Return None if the command is incomplete - - Raise SyntaxError, ValueError or OverflowError if the command is a - syntax error (OverflowError and ValueError can be produced by - malformed literals). + - Raise SyntaxError if the command is a syntax error. """ return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst index 72ac6f630b3d85..1bd43715f49c8d 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-19-23-52.gh-issue-69919.Xx1A2D.rst @@ -1,4 +1,5 @@ :func:`compile`, :func:`exec`, :func:`eval` and :func:`ast.parse` now raise :exc:`SyntaxError` instead of :exc:`ValueError` if the source string contains -surrogate characters, and instead of :exc:`MemoryError` or -:exc:`RecursionError` if the source is too complex to parse or compile. +surrogate characters, instead of :exc:`MemoryError` or :exc:`RecursionError` +if the source is too complex to parse or compile, and instead of +:exc:`OverflowError` if the source is too large. diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 1c85d981ebafde..f4ee83e98e7509 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -68,9 +68,8 @@ _Pypegen_tokenizer_error(Parser *p) break; } case E_COLUMNOVERFLOW: - PyErr_SetString(PyExc_OverflowError, - "Parser column offset overflow - source line is too big"); - return -1; + msg = "source line is too long"; + break; default: msg = "unknown parsing error"; } diff --git a/Parser/string_parser.c b/Parser/string_parser.c index e57460cb1fa648..7bb2979a0554c1 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -295,7 +295,7 @@ _PyPegen_parse_string(Parser *p, Token *t) assert(len >= 1); if (len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "string to parse is too long"); + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "string literal is too long"); return NULL; } if (s[--len] != quote) { diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index 2f2aaf2589246d..51a131f7bbb779 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -146,7 +146,7 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, } if (source->nlines == INT_MAX || (source->nlines == INT_MAX - 1 && newline != NULL)) { - PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines"); + PyErr_SetString(PyExc_SyntaxError, "too many lines in source"); return -1; } return 0; From cf4a5a01b70bbe669e49805c69c96c4aa12d676d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 24 Sep 2026 12:46:41 +0300 Subject: [PATCH 4/4] Update docs and tests changed by GH-157585 Too complex source now raises SyntaxError, so the tests inject MemoryError to test that any compile() error is reported. Compiling a too deeply nested AST object still raises RecursionError. Co-Authored-By: Claude Opus 5.5 (1M context) --- Doc/builtins/functions.rst | 1 + Doc/library/code.rst | 8 +++----- Lib/code.py | 2 +- Lib/test/test_code_module.py | 10 ++++++---- Lib/test/test_pyrepl/test_interact.py | 14 ++++++++------ 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index 6ffeb12b415426..8d9160c0897d3e 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -390,6 +390,7 @@ are always available. They are listed here in alphabetical order. that is too complex to parse or compile, for example an expression with many thousands of nested operators, or that is too large; + :exc:`RecursionError` if an AST object *source* is too deeply nested; and :exc:`ValueError` if *mode* or *flags* is invalid. If you want to parse Python code into its AST representation, see diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 71eb3e8646cd01..70a81b178f26b9 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -74,9 +74,7 @@ build applications which provide an interactive interpreter prompt. Returns a code object (the same as ``compile(source, filename, symbol)``) if the command is complete and valid; ``None`` if the command is incomplete; raises - :exc:`SyntaxError` if the command is complete and contains a syntax error, or - raises :exc:`OverflowError` or :exc:`ValueError` if the command contains an - invalid literal. + :exc:`SyntaxError` if the command is complete and invalid. .. _interpreter-objects: @@ -91,8 +89,8 @@ Interactive Interpreter Objects :func:`compile_command`; the default for *filename* is ``''``, and for *symbol* is ``'single'``. One of several things can happen: - * The input is incorrect; :func:`compile_command` raised an exception - (usually :exc:`SyntaxError`). A syntax traceback will be + * The input is incorrect; :func:`compile_command` raised + :exc:`SyntaxError`. A syntax traceback will be printed by calling the :meth:`showsyntaxerror` method. :meth:`runsource` returns ``False``. diff --git a/Lib/code.py b/Lib/code.py index f658049197e1b7..21fec4c6343679 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -44,7 +44,7 @@ def runsource(self, source, filename="", symbol="single"): One of several things can happen: 1) The input is incorrect; compile_command() raised an - exception (usually SyntaxError). A syntax traceback will be + exception (SyntaxError). A syntax traceback will be printed by calling the showsyntaxerror() method. 2) The input is incomplete, and more input is required; diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 0211e613fc9467..ae019a84a54b06 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -146,13 +146,15 @@ def test_unicode_error(self): def test_compile_error(self): # Any error raised by compile() must be reported (gh-69919). - self.infunc.side_effect = ['-' * 100_000 + '1', EOFError('Finished')] - self.console.interact() + self.infunc.side_effect = ['1', EOFError('Finished')] + with mock.patch.object(self.console, 'compile', + side_effect=MemoryError('spam')): + self.console.interact() output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) output = output[output.index('(InteractiveConsole)'):] output = output[output.index('\n') + 1:] - self.assertRegex(output, r'^(MemoryError|RecursionError): ') - self.assertIn(self.sysmod.last_type, (MemoryError, RecursionError)) + self.assertRegex(output, r'^MemoryError: spam\n') + self.assertIs(self.sysmod.last_type, MemoryError) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_sysexcepthook(self): diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 673827608abc47..1a3dbb5f524d92 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -135,12 +135,13 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): def test_runsource_compile_error(self): # Any error raised by compile() is reported (gh-69919). console = InteractiveColoredConsole() - source = '-' * 100_000 + '1' f = io.StringIO() - with contextlib.redirect_stderr(f): - result = console.runsource(source) + with (contextlib.redirect_stderr(f), + patch.object(console.compile, 'compiler', + side_effect=MemoryError('spam'))): + result = console.runsource('1') self.assertFalse(result) - self.assertRegex(f.getvalue(), r'^(MemoryError|RecursionError): ') + self.assertRegex(f.getvalue(), r'^MemoryError: spam$') def test_runsource_survives_null_bytes(self): console = InteractiveColoredConsole() @@ -195,9 +196,10 @@ def test_invalid_syntax_single_line(self): def test_compile_error_single_line(self): namespace = {} - code = '-' * 100_000 + '1' # MemoryError or RecursionError + code = "1" console = InteractiveColoredConsole(namespace, filename="") - self.assertFalse(_more_lines(console, code)) + with patch.object(console, 'compile', side_effect=MemoryError): + self.assertFalse(_more_lines(console, code)) def test_empty_line(self): namespace = {}