Infra: Add custom lexers for PEP 823 + 824

pep_sphinx_extensions/__init__.py

@@ -14,6 +14,7 @@
     get_from_doctree,
     pep_abstract,
 )
+from pep_sphinx_extensions.lexers import get_custom_lexers
 from pep_sphinx_extensions.pep_processor.html import (
     pep_html_builder,
     pep_html_translator,
@@ -105,6 +106,10 @@ def setup(app: Sphinx) -> dict[str, bool]:
     app.add_directive("superseded", pep_banner_directive.SupersededBanner)
     app.add_directive("withdrawn", pep_banner_directive.WithdrawnBanner)
 
+    # Register custom lexers
+    for lexer in get_custom_lexers():
+        app.add_lexer(lexer.name, lexer)
+
     # Register event callbacks
     app.connect("builder-inited", _update_config_for_builder)  # Update configuration values for builder used
     app.connect("env-before-read-docs", create_pep_zero)  # PEP 0 hook

pep_sphinx_extensions/lexers/__init__.py

@@ -0,0 +1,21 @@
+# This file is placed in the public domain or under the
+# CC0-1.0-Universal license, whichever is more permissive.
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pygments.lexer import Lexer
+
+
+def get_custom_lexers() -> list[type[Lexer]]:
+    lexers: list[type[Lexer]] = []
+    for module_info in pkgutil.walk_packages(__path__, prefix=f"{__name__}."):
+        module = importlib.import_module(module_info.name)
+        if (register_func := getattr(module, "register", None)) is None:
+            continue
+        lexers.extend(register_func())
+    return lexers

pep_sphinx_extensions/lexers/pep823_lexer.py

@@ -0,0 +1,44 @@
+# This file is placed in the public domain or under the
+# CC0-1.0-Universal license, whichever is more permissive.
+
+"""Custom lexer for PEP 823."""
+
+from pygments.lexer import DelegatingLexer, inherit
+from pygments.lexers.python import (
+    PythonLexer,
+    PythonTracebackLexer,
+    _PythonConsoleLexerBase,
+)
+from pygments.token import Operator, Other
+
+
+class Py823Lexer(PythonLexer):
+    name = "py823"
+
+    tokens = {
+        "expr": [
+            (r"maybe\b", Operator.Word),
+            (r"\?", Operator),
+            inherit,
+        ],
+    }
+
+
+class Py823ConsoleLexer(DelegatingLexer):
+    name = "py823-console"
+
+    def __init__(self, **options):
+        pylexer = Py823Lexer
+        tblexer = PythonTracebackLexer
+
+        class _ReplaceInnerCode(DelegatingLexer):
+            def __init__(self, **options):
+                super().__init__(
+                    pylexer, _PythonConsoleLexerBase, Other.Code, **options
+                )
+
+        super().__init__(tblexer, _ReplaceInnerCode, Other.Traceback, **options)
+
+
+def register():
+    return [Py823Lexer, Py823ConsoleLexer]

pep_sphinx_extensions/lexers/pep824_lexer.py

@@ -0,0 +1,25 @@
+# This file is placed in the public domain or under the
+# CC0-1.0-Universal license, whichever is more permissive.
+
+"""Custom lexer for PEP 824."""
+
+from pygments.lexer import inherit
+from pygments.lexers.python import PythonLexer
+from pygments.token import Operator
+
+
+class Py824Lexer(PythonLexer):
+    name = "py824"
+
+    tokens = {
+        "expr": [
+            (r"\?\?(?=\s)", Operator.Word),
+            (r"otherwise\b", Operator.Word),
+            (r"\?", Operator),
+            inherit,
+        ],
+    }
+
+
+def register():
+    return [Py824Lexer]