Allow FilterStack to use a custom lexer

sqlparse/engine/filter_stack.py

@@ -2,8 +2,8 @@
 # Copyright (C) 2009-2020 the sqlparse authors and contributors
 # <see AUTHORS file>
 #
-# This module is part of python-sqlparse and is released under
-# the BSD License: https://opensource.org/licenses/BSD-3-Clause
+# This module is part of python-sqlparse and is released under the BSD License:
+# https://opensource.org/licenses/BSD-3-Clause
 
 """filter"""
 
@@ -20,6 +20,7 @@ def __init__(self, strip_semicolon=False):
         self.stmtprocess = []
         self.postprocess = []
         self._grouping = False
+        self.lexer = lexer.Lexer.get_default_instance()
         if strip_semicolon:
             self.stmtprocess.append(StripTrailingSemicolonFilter())
 
@@ -28,7 +29,7 @@ def enable_grouping(self):
 
     def run(self, sql, encoding=None):
         try:
-            stream = lexer.tokenize(sql, encoding)
+            stream = self.lexer.get_tokens(sql, encoding)
             # Process token stream
             for filter_ in self.preprocess:
                 stream = filter_.process(stream)

tests/test_filter_stack.py

@@ -0,0 +1,14 @@
+from sqlparse import engine, tokens
+from sqlparse.lexer import Lexer
+
+
+def test_filter_stack_allows_custom_lexer():
+    custom_lexer = Lexer()
+    custom_lexer.default_initialization()
+    custom_lexer.add_keywords({'CUSTOMLEXERKEYWORD': tokens.Keyword})
+
+    stack = engine.FilterStack()
+    stack.lexer = custom_lexer
+    statement = next(stack.run('CUSTOMLEXERKEYWORD value'))
+
+    assert statement.tokens[0].ttype is tokens.Keyword