diff --git a/lark/lark.py b/lark/lark.py index a32c214..fb365de 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -185,7 +185,7 @@ class LarkOptions(Serialize): # Options that can be passed to the Lark parser, even when it was loaded from cache/standalone. # These option are only used outside of `load_grammar`. -_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'} +_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'} _VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None) _VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest') diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index eea582a..7061750 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -43,9 +43,8 @@ class MakeParsingFrontend: def deserialize_lexer_conf(cls, data, memo, options): # We need lexer_conf earley to have the terminals that we need to produce the callback list for paser_conf # So we split deserialize into two methods - terminals = [item for item in memo.values() if isinstance(item, TerminalDef)] lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo) - lexer_conf.callbacks = _get_lexer_callbacks(options.transformer, terminals) + lexer_conf.callbacks = options.lexer_callbacks or {} lexer_conf.re_module = regex if options.regex else re lexer_conf.use_bytes = options.use_bytes lexer_conf.g_regex_flags = options.g_regex_flags diff --git a/tests/test_cache.py b/tests/test_cache.py index 61af293..80c7609 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -55,6 +55,18 @@ class CustomLexer(Lexer): yield Token('A', obj) +class TestT(Transformer): + def add(self, children): + return sum(children if isinstance(children, list) else children.children) + + def NUM(self, token): + return int(token) + + +def append_zero(t): + return t.update(value=t.value + '0') + + class TestCache(TestCase): def setUp(self): pass @@ -80,7 +92,7 @@ class TestCache(TestCase): parser = Lark(g, parser='lalr', cache=True) assert parser.parse('a') == Tree('start', []) - parser = Lark(g+' "b"', parser='lalr', cache=True) + parser = Lark(g + ' "b"', parser='lalr', cache=True) assert len(mock_fs.files) == 2 assert parser.parse('ab') == Tree('start', []) @@ -100,7 +112,7 @@ class TestCache(TestCase): parser = Lark(g, parser="lalr", debug=True, cache=True) assert parser.options.options['debug'] - # Test inline transformer (tree-less) + # Test inline transformer (tree-less) & lexer_callbacks mock_fs.files = {} g = """ start: add+ @@ -109,19 +121,19 @@ class TestCache(TestCase): %ignore " " """ text = "1+2 3+4" - expected = Tree('start', [3, 7]) + expected = Tree('start', [30, 70]) - parser = Lark(g, parser='lalr', transformer=TestT(), cache=True) - parser = Lark(g, parser='lalr', transformer=TestT(), cache=True) + parser = Lark(g, parser='lalr', transformer=TestT(), cache=True, lexer_callbacks={'NUM': append_zero}) + res0 = parser.parse(text) + parser = Lark(g, parser='lalr', transformer=TestT(), cache=True, lexer_callbacks={'NUM': append_zero}) assert len(mock_fs.files) == 1 res1 = parser.parse(text) - res2 = TestT().transform( Lark(g, parser="lalr", cache=True).parse(text) ) - assert res1 == res2 + res2 = TestT().transform(Lark(g, parser="lalr", cache=True, lexer_callbacks={'NUM': append_zero}).parse(text)) + assert res0 == res1 == res2 == expected finally: lark_module.FS = fs - if __name__ == '__main__': main()