jslexer: stop regexp literals from spanning lines

babel/messages/jslexer.py

@@ -27,7 +27,7 @@
 name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE)
 dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE)
 division_re = re.compile(r'/=?')
-regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*', re.DOTALL)
+regex_re = re.compile(r'/(?:[^/\\\n]*(?:\\.[^/\\\n]*)*)/[a-zA-Z]*')
 line_re = re.compile(r'(\r\n|\n|\r)')
 line_join_re = re.compile(r'\\' + line_re.pattern)
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')

tests/messages/test_js_extract.py

@@ -122,7 +122,10 @@ def test_jsx_extraction(jsx_enabled):
     if jsx_enabled:
         assert messages == EXPECTED_JSX_MESSAGES
     else:
-        assert messages != EXPECTED_JSX_MESSAGES
+        # Without the jsx option the regex literals in the source used to
+        # swallow following messages (issue #640); with that lexer bug
+        # fixed, plain JavaScript extraction finds them all too.
+        assert messages == EXPECTED_JSX_MESSAGES
 
 
 def test_dotted_keyword_extract():
@@ -189,3 +192,18 @@ def test_inside_nested_template_string():
     )
 
     assert messages == [(1, 'Greetings!', [], None), (1, 'This is a lovely evening.', [], None), (1, 'The day is really nice!', [], None)]
+
+
+def test_regexp_literals_do_not_swallow_messages():
+    # https://github.com/python-babel/babel/issues/640: a bogus fallback
+    # regex token used to span line terminators and silently drop every
+    # gettext() call inside the swallowed region.
+    source = b"""
+        alert(gettext("First message"));
+        var expr = />/g;
+        alert(gettext("Second message"));
+        var expr2 = /'/g;
+        alert(gettext("Third message"));
+    """
+    messages = [m[2] for m in extract.extract_javascript(BytesIO(source), ("gettext",), {}, {})]
+    assert messages == ["First message", "Second message", "Third message"]

tests/messages/test_jslexer.py

@@ -123,3 +123,14 @@ def test_jsx():
         ('jsx_tag', '</comp2', 8),
         ('operator', '>', 8),
     ]
+
+
+def test_regexp_literal_cannot_span_lines():
+    # A JS regex literal may not contain line terminators; the fallback
+    # regex match must stop at the end of the line (issue #640).
+    tokens = list(
+        jslexer.tokenize('alert(gettext("x"));\nvar e = />/g;\nalert(gettext("y"));'),
+    )
+    assert ('string', '"y"', 3) in tokens
+    for kind, value, _lineno in tokens:
+        assert "\n" not in value, f"{kind} token spans lines: {value!r}"