diff --git a/sqlparse/engine/filter_stack.py b/sqlparse/engine/filter_stack.py index 415d3fc9..27153cd0 100644 --- a/sqlparse/engine/filter_stack.py +++ b/sqlparse/engine/filter_stack.py @@ -2,8 +2,8 @@ # Copyright (C) 2009-2020 the sqlparse authors and contributors # # -# This module is part of python-sqlparse and is released under -# the BSD License: https://opensource.org/licenses/BSD-3-Clause +# This module is part of python-sqlparse and is released under the BSD License: +# https://opensource.org/licenses/BSD-3-Clause """filter""" @@ -20,6 +20,7 @@ def __init__(self, strip_semicolon=False): self.stmtprocess = [] self.postprocess = [] self._grouping = False + self.lexer = lexer.Lexer.get_default_instance() if strip_semicolon: self.stmtprocess.append(StripTrailingSemicolonFilter()) @@ -28,7 +29,7 @@ def enable_grouping(self): def run(self, sql, encoding=None): try: - stream = lexer.tokenize(sql, encoding) + stream = self.lexer.get_tokens(sql, encoding) # Process token stream for filter_ in self.preprocess: stream = filter_.process(stream) diff --git a/tests/test_filter_stack.py b/tests/test_filter_stack.py new file mode 100644 index 00000000..c98672ac --- /dev/null +++ b/tests/test_filter_stack.py @@ -0,0 +1,14 @@ +from sqlparse import engine, tokens +from sqlparse.lexer import Lexer + + +def test_filter_stack_allows_custom_lexer(): + custom_lexer = Lexer() + custom_lexer.default_initialization() + custom_lexer.add_keywords({'CUSTOMLEXERKEYWORD': tokens.Keyword}) + + stack = engine.FilterStack() + stack.lexer = custom_lexer + statement = next(stack.run('CUSTOMLEXERKEYWORD value')) + + assert statement.tokens[0].ttype is tokens.Keyword