Fix for #818 + tests

lark/lark.py

@@ -338,9 +338,7 @@ class Lark(Serialize):
                     rule.options.priority = None
 
         # TODO Deprecate lexer_callbacks?
-        lexer_callbacks = (_get_lexer_callbacks(self.options.transformer, self.terminals)
-                           if self.options.transformer
-                           else {})
+        lexer_callbacks = {}
         lexer_callbacks.update(self.options.lexer_callbacks)
 
         self.lexer_conf = LexerConf(self.terminals, re_module, self.ignore_tokens, self.options.postlex, lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes)
@@ -370,7 +368,7 @@ class Lark(Serialize):
 
     def _prepare_callbacks(self):
         self.parser_class = get_frontend(self.options.parser, self.options.lexer)
-        self._callbacks = None
+        self._callbacks = {}
         # we don't need these callbacks if we aren't building a tree
         if self.options.ambiguity != 'forest':
             self._parse_tree_builder = ParseTreeBuilder(
@@ -380,7 +378,8 @@ class Lark(Serialize):
                     self.options.parser != 'lalr' and self.options.ambiguity == 'explicit',
                     self.options.maybe_placeholders
                 )
-            self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
+            self._callbacks.update(self._parse_tree_builder.create_callback(self.options.transformer))
+        self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals))
 
     def _build_parser(self):
         self._prepare_callbacks()

lark/parsers/lalr_parser.py

@@ -101,7 +101,7 @@ class ParserState(object):
                 # shift once and return
                 assert not is_end
                 state_stack.append(arg)
-                value_stack.append(token)
+                value_stack.append(token if token.type not in callbacks else callbacks[token.type](token))
                 return
             else:
                 # reduce+shift as many times as necessary

tests/test_parser.py

@@ -10,7 +10,7 @@ from copy import copy, deepcopy
 
 from lark.utils import Py36, isascii
 
-from lark import Token
+from lark import Token, Transformer_NonRecursive
 
 try:
     from cStringIO import StringIO as cStringIO
@@ -34,7 +34,7 @@ from lark import logger
 from lark.lark import Lark
 from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters
 from lark.tree import Tree
-from lark.visitors import Transformer, Transformer_InPlace, v_args
+from lark.visitors import Transformer, Transformer_InPlace, v_args, Transformer_InPlaceRecursive
 from lark.grammar import Rule
 from lark.lexer import TerminalDef, Lexer, TraditionalLexer
 from lark.indenter import Indenter
@@ -162,6 +162,28 @@ class TestParsers(unittest.TestCase):
         r = p.parse("x")
         self.assertEqual( r.children, ["X!"] )
 
+    def test_visit_tokens2(self):
+        g = """
+        start: add+
+        add: NUM "+" NUM
+        NUM: /\d+/
+        %ignore " "
+        """
+        text = "1+2 3+4"
+        expected = Tree('start', [3, 7])
+        for base in (Transformer, Transformer_InPlace, Transformer_NonRecursive, Transformer_InPlaceRecursive):
+            class T(base):
+                def add(self, children):
+                    return sum(children)
+                
+                def NUM(self, token):
+                    return int(token)
+                
+            
+            parser = Lark(g, parser='lalr', transformer=T())
+            result = parser.parse(text)
+            self.assertEqual(result, expected)
+
     def test_vargs_meta(self):
 
         @v_args(meta=True)