From a4fac1773d09512eb83912bcc55859f8a7f40829 Mon Sep 17 00:00:00 2001 From: Rami Abdelrazzaq Date: Sat, 12 Sep 2026 05:16:57 -0500 Subject: [PATCH 1/2] Allow FilterStack to use a custom lexer --- sqlparse/engine/filter_stack.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sqlparse/engine/filter_stack.py b/sqlparse/engine/filter_stack.py index 415d3fc9..27153cd0 100644 --- a/sqlparse/engine/filter_stack.py +++ b/sqlparse/engine/filter_stack.py @@ -2,8 +2,8 @@ # Copyright (C) 2009-2020 the sqlparse authors and contributors # # -# This module is part of python-sqlparse and is released under -# the BSD License: https://opensource.org/licenses/BSD-3-Clause +# This module is part of python-sqlparse and is released under the BSD License: +# https://opensource.org/licenses/BSD-3-Clause """filter""" @@ -20,6 +20,7 @@ def __init__(self, strip_semicolon=False): self.stmtprocess = [] self.postprocess = [] self._grouping = False + self.lexer = lexer.Lexer.get_default_instance() if strip_semicolon: self.stmtprocess.append(StripTrailingSemicolonFilter()) @@ -28,7 +29,7 @@ def enable_grouping(self): def run(self, sql, encoding=None): try: - stream = lexer.tokenize(sql, encoding) + stream = self.lexer.get_tokens(sql, encoding) # Process token stream for filter_ in self.preprocess: stream = filter_.process(stream) From a92901e85608e9009491973e43e8a4fe98a87ef0 Mon Sep 17 00:00:00 2001 From: Rami Abdelrazzaq Date: Sat, 12 Sep 2026 05:17:19 -0500 Subject: [PATCH 2/2] Test custom lexer support in FilterStack --- tests/test_filter_stack.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/test_filter_stack.py diff --git a/tests/test_filter_stack.py b/tests/test_filter_stack.py new file mode 100644 index 00000000..c98672ac --- /dev/null +++ b/tests/test_filter_stack.py @@ -0,0 +1,14 @@ +from sqlparse import engine, tokens +from sqlparse.lexer import Lexer + + +def test_filter_stack_allows_custom_lexer(): + custom_lexer = Lexer() + custom_lexer.default_initialization() + custom_lexer.add_keywords({'CUSTOMLEXERKEYWORD': tokens.Keyword}) + + stack = engine.FilterStack() + stack.lexer = custom_lexer + statement = next(stack.run('CUSTOMLEXERKEYWORD value')) + + assert statement.tokens[0].ttype is tokens.Keyword