Merge pull request #759 from MegaIng/fix-758

Make `LALR_CustomLexerWrapper` behave correctly + tests

lark/parser_frontends.py

@@ -29,10 +29,9 @@ def get_frontend(parser, lexer):
                 def lex(self, lexer_state, parser_state):
                     return self.lexer.lex(lexer_state.text)
 
-            class LALR_CustomLexerWrapper(LALR_CustomLexer):
+            class LALR_CustomLexerWrapper(LALR_WithLexer):
                 def __init__(self, lexer_conf, parser_conf, options=None):
-                    super(LALR_CustomLexerWrapper, self).__init__(
-                        lexer, lexer_conf, parser_conf, options=options)
+                    super(LALR_CustomLexerWrapper, self).__init__(lexer_conf, parser_conf, options=options)
                 def init_lexer(self):
                     future_interface = getattr(lexer, '__future_interface__', False)
                     if future_interface:
@@ -163,13 +162,6 @@ class LALR_ContextualLexer(LALR_WithLexer):
 
 ###}
 
-class LALR_CustomLexer(LALR_WithLexer):
-    def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None):
-        self.lexer = lexer_cls(lexer_conf)
-        debug = options.debug if options else False
-        self.parser = LALR_Parser(parser_conf, debug=debug)
-        WithLexer.__init__(self, lexer_conf, parser_conf, options)
-
 
 class Earley(WithLexer):
     def __init__(self, lexer_conf, parser_conf, options=None):

tests/test_parser.py

@@ -322,6 +322,22 @@ class TestParsers(unittest.TestCase):
 
     def test_alias(self):
         Lark("""start: ["a"] "b" ["c"] "e" ["f"] ["g"] ["h"] "x" -> d """)
+        
+    def test_backwards_custom_lexer(self):
+        class OldCustomLexer(Lexer):
+            def __init__(self, lexer_conf):
+                pass
+
+            def lex(self, text):
+                yield Token('A', 'A')
+        
+        p = Lark("""
+        start: A
+        %declare A
+        """, parser='lalr', lexer=OldCustomLexer)
+        
+        r = p.parse('')
+        self.assertEqual(r, Tree('start', [Token('A', 'A')]))
 
 
 
@@ -849,6 +865,8 @@ class CustomLexer(Lexer):
         self.lexer = TraditionalLexer(copy(lexer_conf))
     def lex(self, *args, **kwargs):
         return self.lexer.lex(*args, **kwargs)
+    
+    __future_interface__ = True
 
 def _tree_structure_check(a, b):
     """