diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py index d751b58f7..3504ee564 100644 --- a/babel/messages/jslexer.py +++ b/babel/messages/jslexer.py @@ -27,7 +27,7 @@ name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE) dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE) division_re = re.compile(r'/=?') -regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*', re.DOTALL) +regex_re = re.compile(r'/(?:[^/\\\n]*(?:\\.[^/\\\n]*)*)/[a-zA-Z]*') line_re = re.compile(r'(\r\n|\n|\r)') line_join_re = re.compile(r'\\' + line_re.pattern) uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}') diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py index af6dfb144..c45eeb38b 100644 --- a/tests/messages/test_js_extract.py +++ b/tests/messages/test_js_extract.py @@ -122,7 +122,10 @@ def test_jsx_extraction(jsx_enabled): if jsx_enabled: assert messages == EXPECTED_JSX_MESSAGES else: - assert messages != EXPECTED_JSX_MESSAGES + # Without the jsx option the regex literals in the source used to + # swallow following messages (issue #640); with that lexer bug + # fixed, plain JavaScript extraction finds them all too. + assert messages == EXPECTED_JSX_MESSAGES def test_dotted_keyword_extract(): @@ -189,3 +192,18 @@ def test_inside_nested_template_string(): ) assert messages == [(1, 'Greetings!', [], None), (1, 'This is a lovely evening.', [], None), (1, 'The day is really nice!', [], None)] + + +def test_regexp_literals_do_not_swallow_messages(): + # https://github.com/python-babel/babel/issues/640: a bogus fallback + # regex token used to span line terminators and silently drop every + # gettext() call inside the swallowed region. + source = b""" + alert(gettext("First message")); + var expr = />/g; + alert(gettext("Second message")); + var expr2 = /'/g; + alert(gettext("Third message")); + """ + messages = [m[2] for m in extract.extract_javascript(BytesIO(source), ("gettext",), {}, {})] + assert messages == ["First message", "Second message", "Third message"] diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py index 00afc78ae..c28d8fee0 100644 --- a/tests/messages/test_jslexer.py +++ b/tests/messages/test_jslexer.py @@ -123,3 +123,14 @@ def test_jsx(): ('jsx_tag', '', 8), ] + + +def test_regexp_literal_cannot_span_lines(): + # A JS regex literal may not contain line terminators; the fallback + # regex match must stop at the end of the line (issue #640). + tokens = list( + jslexer.tokenize('alert(gettext("x"));\nvar e = />/g;\nalert(gettext("y"));'), + ) + assert ('string', '"y"', 3) in tokens + for kind, value, _lineno in tokens: + assert "\n" not in value, f"{kind} token spans lines: {value!r}"