From a202f2c7e8318bf582a6a75b4379c2efe58f2fad Mon Sep 17 00:00:00 2001
From: MegaIng1 <trampchamp@hotmail.de>
Date: Thu, 25 Mar 2021 18:56:37 +0100
Subject: [PATCH] Fix for cache + inline Transformer

---
 lark/lark.py             |  2 +-
 lark/parser_frontends.py |  3 +--
 tests/test_cache.py      | 39 ++++++++++++++++++++++++++++++++++++---
 3 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/lark/lark.py b/lark/lark.py
index c0419ce..c6f3f7e 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -185,7 +185,7 @@ class LarkOptions(Serialize):
 
 # Options that can be passed to the Lark parser, even when it was loaded from cache/standalone.
 # These option are only used outside of `load_grammar`.
-_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
+_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
 
 _VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None)
 _VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest')
diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py
index c5049e9..e9d89b0 100644
--- a/lark/parser_frontends.py
+++ b/lark/parser_frontends.py
@@ -43,9 +43,8 @@ class MakeParsingFrontend:
     def deserialize_lexer_conf(cls, data, memo, options):
         # We need lexer_conf earley to have the terminals that we need to produce the callback list for paser_conf
         # So we split deserialize into two methods
-        terminals = [item for item in memo.values() if isinstance(item, TerminalDef)]
         lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo)
-        lexer_conf.callbacks = _get_lexer_callbacks(options.transformer, terminals)
+        lexer_conf.callbacks = options.lexer_callbacks or {}
         lexer_conf.re_module = regex if options.regex else re
         lexer_conf.use_bytes = options.use_bytes
         lexer_conf.g_regex_flags = options.g_regex_flags
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 4a07d7a..d5a92c8 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -3,7 +3,7 @@ from __future__ import absolute_import
 import sys
 from unittest import TestCase, main
 
-from lark import Lark, Tree
+from lark import Lark, Tree, Transformer
 from lark.lexer import Lexer, Token
 import lark.lark as lark_module
 
@@ -48,6 +48,18 @@ class CustomLexer(Lexer):
             yield Token('A', obj)
 
 
+class TestT(Transformer):
+    def add(self, children):
+        return sum(children if isinstance(children, list) else children.children)
+
+    def NUM(self, token):
+        return int(token)
+
+
+def append_zero(t):
+    return t.update(value=t.value + '0')
+
+
 class TestCache(TestCase):
     def setUp(self):
         pass
@@ -73,7 +85,7 @@ class TestCache(TestCase):
             parser = Lark(g, parser='lalr', cache=True)
             assert parser.parse('a') == Tree('start', [])
 
-            parser = Lark(g+' "b"', parser='lalr', cache=True)
+            parser = Lark(g + ' "b"', parser='lalr', cache=True)
             assert len(mock_fs.files) == 2
             assert parser.parse('ab') == Tree('start', [])
 
@@ -92,10 +104,31 @@ class TestCache(TestCase):
             Lark(g, parser="lalr", debug=True, cache=True)
             parser = Lark(g, parser="lalr", debug=True, cache=True)
             assert parser.options.options['debug']
+
+            # Test inline transformer (tree-less) & lexer_callbacks
+            mock_fs.files = {}
+            g = """
+            start: add+
+            add: NUM "+" NUM
+            NUM: /\d+/
+            %ignore " "
+            """
+            text = "1+2 3+4"
+            expected = Tree('start', [30, 70])
+
+            parser = Lark(g, parser='lalr', transformer=TestT(), cache=True, lexer_callbacks={'NUM': append_zero})
+            res0 = parser.parse(text)
+            parser = Lark(g, parser='lalr', transformer=TestT(), cache=True, lexer_callbacks={'NUM': append_zero})
+            assert len(mock_fs.files) == 1
+            res1 = parser.parse(text)
+            res2 = TestT().transform(Lark(g, parser="lalr", cache=True, lexer_callbacks={'NUM': append_zero}).parse(text))
+            assert res0 == res1 == res2 == expected
+
+
+
         finally:
             lark_module.FS = fs
 
 
-
 if __name__ == '__main__':
     main()