Merge remote-tracking branch 'upstream/3.15' into mi-155672

Author: serhiy-storchaka

Doc/builtins/functions.rst

@@ -344,8 +344,14 @@ are always available.  They are listed here in alphabetical order.
    It is needed to unambiguous :ref:`filter <warning-filter>` syntax warnings
    by module name.
 
-   This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled
-   source is invalid.
+   This function raises :exc:`SyntaxError` if the compiled source is invalid,
+   including a *source* containing a null character or that cannot be decoded;
+   :exc:`ValueError` if *mode* or *flags* is invalid,
+   or if a string *source* contains surrogate characters;
+   :exc:`MemoryError` or :exc:`RecursionError` if *source* is too complex
+   to parse or compile,
+   for example an expression with many thousands of nested operators;
+   and :exc:`OverflowError` if *source* is too large.
 
    If you want to parse Python code into its AST representation, see
    :func:`ast.parse`.
@@ -377,11 +383,15 @@ are always available.  They are listed here in alphabetical order.
       Previously, :exc:`TypeError` was raised when null bytes were encountered
       in *source*.
 
-   .. versionadded:: 3.8
+   .. versionchanged:: 3.8
       ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
       support for top-level ``await``, ``async for``, and ``async with``.
 
-   .. versionadded:: 3.15
+   .. versionchanged:: 3.12
+      :exc:`SyntaxError` is raised instead of :exc:`ValueError` when null bytes
+      are encountered in *source*.
+
+   .. versionchanged:: 3.15
       Added the *module* parameter.
 
 

Doc/library/code.rst

@@ -92,7 +92,7 @@ Interactive Interpreter Objects
    *symbol* is ``'single'``.  One of several things can happen:
 
    * The input is incorrect; :func:`compile_command` raised an exception
-     (:exc:`SyntaxError` or :exc:`OverflowError`).  A syntax traceback will be
+     (usually :exc:`SyntaxError`).  A syntax traceback will be
      printed by calling the :meth:`showsyntaxerror` method.  :meth:`runsource`
      returns ``False``.
 

Lib/_pyrepl/console.py

@@ -261,7 +261,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
                 )
             self.showsyntaxerror(filename, source=source)
             return False
-        except (OverflowError, ValueError):
+        except Exception:
             self.showsyntaxerror(filename, source=source)
             return False
         if tree.body:
@@ -282,7 +282,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
                     )
                 self.showsyntaxerror(filename, source=source)
                 return False
-            except (OverflowError, ValueError):
+            except Exception:
                 self.showsyntaxerror(filename, source=source)
                 return False
 

Lib/_pyrepl/simple_interact.py

@@ -83,7 +83,7 @@ def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool:
     src = _strip_final_indent(unicodetext)
     try:
         code = console.compile(src, "<stdin>", "single")
-    except (OverflowError, SyntaxError, ValueError):
+    except Exception:
         lines = src.splitlines(keepends=True)
         if len(lines) == 1:
             return False

Lib/code.py

@@ -44,8 +44,8 @@ def runsource(self, source, filename="<input>", symbol="single"):
         One of several things can happen:
 
         1) The input is incorrect; compile_command() raised an
-        exception (SyntaxError or OverflowError).  A syntax traceback
-        will be printed by calling the showsyntaxerror() method.
+        exception (usually SyntaxError).  A syntax traceback will be
+        printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
@@ -62,7 +62,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
         """
         try:
             code = self.compile(source, filename, symbol)
-        except (OverflowError, SyntaxError, ValueError):
+        except Exception:
             # Case 1
             self.showsyntaxerror(filename, source=source)
             return False

Lib/idlelib/__main__.py

@@ -3,5 +3,12 @@
 
 Run IDLE as python -m idlelib
 """
+import sys
+
+if not sys.flags.safe_path:
+    # Remove the current directory, prepended by "python -m", so that
+    # user files do not shadow IDLE's imports (gh-70331).
+    del sys.path[0]
+
 import idlelib.pyshell
 idlelib.pyshell.main()

Lib/idlelib/colorizer.py

@@ -8,6 +8,11 @@
 
 DEBUG = False
 
+# Adding a tag to a line takes time proportional to the number of tags
+# already in the line, so only the beginning of a line is colorized;
+# the rest is usually not visible anyway (gh-103089).
+MAX_COLORIZED_LINE = 2000
+
 
 def any(name, alternates):
     "Return a named group pattern matching list of alternates."
@@ -42,6 +47,11 @@ def make_pat():
         ]) +
         r"))"
     )
+    type_softkw = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<TYPE_SOFTKW>type)" +
+        r"(?=[ \t]+(?!(?:" + "|".join(keyword.kwlist) + r")\b)[^\W\d])"
+    )
     lazy_softkw = (  # lazy new in 3.15 (+ 2 lines below).
         r"^[ \t]*" +  # at beginning of line + possible indentation
         r"(?P<LAZY_SOFTKW>lazy)" +
@@ -59,7 +69,7 @@ def make_pat():
     dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
     string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
     prog = re.compile("|".join([
-                                builtin, comment, string, kw,
+                                type_softkw, builtin, comment, string, kw,
                                 match_softkw, case_default,
                                 case_softkw_and_pattern, lazy_softkw,
                                 any("SYNC", [r"\n"]),
@@ -75,6 +85,7 @@ def make_pat():
     "CASE_SOFTKW": "KEYWORD",
     "CASE_DEFAULT_UNDERSCORE": "KEYWORD",
     "CASE_SOFTKW2": "KEYWORD",
+    "TYPE_SOFTKW": "KEYWORD",
     "LAZY_SOFTKW": "KEYWORD",
 }
 
@@ -344,16 +355,39 @@ def _add_tags_in_section(self, chars, head):
         `chars` is a string with the text to parse and to which
         highlighting is to be applied.
 
-            `head` is the index in the text widget where the text is found.
+        `head` is the index in the text widget where the text is found.
         """
-        for m in self.prog.finditer(chars):
+        # Positions are relative to the start of the current line, so that
+        # Tk does not resolve them through the previous lines.
+        line = int(head.split('.')[0])
+        line_start = 0  # Offset of the current line in chars.
+        tags = []
+        pos = 0
+        while True:
+            m = self.prog.search(chars, pos)
+            if m is None:
+                break
             for name, matched_text in matched_named_groups(m):
                 a, b = m.span(name)
-                self._add_tag(a, b, head, name)
+                tags.append((a - line_start, b - line_start, head, name))
                 if matched_text in ("def", "class"):
                     if m1 := self.idprog.match(chars, b):
                         a, b = m1.span(1)
-                        self._add_tag(a, b, head, "DEFINITION")
+                        tags.append((a - line_start, b - line_start,
+                                     head, "DEFINITION"))
+            pos = m.end()
+            if '\n' in m[0]:
+                line += m[0].count('\n')
+                line_start = m.start() + m[0].rindex('\n') + 1
+                head = f"{line}.0"
+            elif pos - line_start >= MAX_COLORIZED_LINE:
+                # The rest of a long line is not colorized.
+                pos = chars.find('\n', pos)
+                if pos < 0:
+                    break
+        # Adding a tag is faster if there are no tags after it.
+        for args in reversed(tags):
+            self._add_tag(*args)
 
     def removecolors(self):
         "Remove all colorizing tags."

Lib/idlelib/editor.py

@@ -1120,19 +1120,50 @@ def _close(self):
 
     def last_mtime(self):
         file = self.io.filename
-        return os.path.getmtime(file) if file else 0
+        if not file:
+            return None
+        try:
+            return os.path.getmtime(file)
+        except OSError:
+            # File is gone or cannot be stat'ed.
+            return None
 
     def focus_in_event(self, event):
         mtime = self.last_mtime()
-        if self.mtime != mtime:
+        if mtime == self.mtime:
+            return
+        if self.mtime is not None and mtime is None:
+            # The file was there and is now gone; reloading cannot work.
+            self.deleted_file_event(event)
+        else:
             self.mtime = mtime
-            if self. askyesno(
+            if self.askyesno(
               'Reload', '"%s"\n\nThis script has been modified by another program.'
               '\nDo you want to reload it?' % self.io.filename, parent=self.text):
                 self.io.loadfile(self.io.filename)
             else:
                 self.set_saved(False)
 
+    def deleted_file_event(self, event):
+        # The file was deleted or renamed while open; ask what to do with the
+        # buffer instead of offering a reload that could only fail.  Forget the
+        # old mtime before showing the dialog so a FocusIn delivered while this
+        # dialog (or a Close/Save As sub-dialog) is open does not reopen it; a
+        # successful Save As restores it via set_saved(True).
+        self.mtime = None
+        dialog = simpledialog.SimpleDialog(
+            self.text,
+            title='File Deleted',
+            text='"%s"\n\nThis file no longer exists.' % self.io.filename,
+            buttons=('Close', 'Save As', 'Ignore'),
+            default=1,
+            cancel=2)
+        choice = dialog.go()
+        if choice == 0:
+            self.close()
+        elif choice == 1:
+            self.io.save_as(event)
+
     def load_extensions(self):
         self.extensions = {}
         self.load_standard_extensions()

Lib/idlelib/idle.py

@@ -1,6 +1,12 @@
-import os.path
 import sys
 
+if __spec__ is not None and not sys.flags.safe_path:
+    # Remove the current directory, prepended by "python -m", so that
+    # user files do not shadow IDLE's imports (gh-70331).
+    del sys.path[0]
+
+import os.path
+
 
 # Enable running IDLE with idlelib in a non-standard location.
 # This was once used to run development versions of IDLE.

Lib/idlelib/idle_test/test_colorizer.py

@@ -52,6 +52,8 @@ async def f(): await g()
     '''
     case _:'''
     "match x:"
+    type Point = tuple[float, float]
+    type = type(1)
     """)
 
 
@@ -404,6 +406,8 @@ def test_recolorize_main(self, mock_notify):
                     ('28.25', ('STRING',)), ('28.38', ('STRING',)),
                     ('30.0', ('STRING',)),
                     ('31.1', ('STRING',)),
+                    ('32.0', ('KEYWORD',)),
+                    ('33.0', ('BUILTIN',)), ('33.1', ('BUILTIN',)),
                     # SYNC at the end of every line.
                     ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)),
                    )
@@ -434,7 +438,7 @@ def test_recolorize_main(self, mock_notify):
         eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17'))
         eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26'))
         eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0'))
-        eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '33.0'))
+        eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '32.0'))
 
     def _assert_highlighting(self, source, tag_ranges):
         """Check highlighting of a given piece of code.
@@ -569,6 +573,18 @@ def test_long_multiline_string(self):
             e"""
             ''')
         self._assert_highlighting(source, {'STRING': [('1.0', '5.4')]})
+        source = '"""a\nb""" + str\n'
+        self._assert_highlighting(source, {'STRING': [('1.0', '2.4')],
+                                           'BUILTIN': [('2.7', '2.10')]})
+
+    def test_long_line(self):
+        # gh-103089: only the first MAX_COLORIZED_LINE characters of a line
+        # are colorized.
+        n = colorizer.MAX_COLORIZED_LINE
+        source = f"pass\n{'x' * (n - 3)}'a', 'b'\n'c'\n"
+        self._assert_highlighting(source, {'KEYWORD': [('1.0', '1.4')],
+                                           'STRING': [(f'2.{n-3}', f'2.{n}'),
+                                                      ('3.0', '3.3')]})
 
     @run_in_tk_mainloop(delay=50)
     def test_incremental_editing(self):