Refactored puppet + small fixes

lark/parsers/lalr_parser.py

@@ -7,9 +7,10 @@ from ..lexer import Token
 from ..utils import Enumerator, Serialize
 
 from .lalr_analysis import LALR_Analyzer, Shift, Reduce, IntParseTable
-
+from .lalr_puppet import ParserPuppet
 
 ###{standalone
+
 class LALR_Parser(object):
     def __init__(self, parser_conf, debug=False):
         assert all(r.options.priority is None for r in parser_conf.rules), "LALR doesn't yet support prioritization"
@@ -59,7 +60,11 @@ class _Parser:
                 return states[state][token.type]
             except KeyError:
                 expected = [s for s in states[state].keys() if s.isupper()]
-                raise UnexpectedToken(token, expected, state=state, puppet=_ParserPuppet(self, state_stack, value_stack, start, stream, set_state))
+                try:
+                    puppet = ParserPuppet(self, state_stack, value_stack, start, stream, set_state)
+                except NameError:
+                    puppet = None
+                raise UnexpectedToken(token, expected, state=state, puppet=puppet)
 
         def reduce(rule):
             size = len(rule.expansion)
@@ -112,58 +117,3 @@ class _Parser:
 
 ###}
 
-
-
-
-class _ParserPuppet:
-    def __init__(self, parser, state_stack, value_stack, start, stream, set_state):
-        self.parser = parser
-        self._state_stack = state_stack
-        self._value_stack = value_stack
-        self._start = start
-        self._stream = stream
-        self._set_state = set_state
-
-    def feed_token(self, token):
-        end_state = self.parser.parse_table.end_states[self._start]
-        state_stack = self._state_stack
-        value_stack = self._value_stack
-
-        state = state_stack[-1]
-        action, arg = self.parser.parse_table.states[state][token.type]
-        assert arg != end_state
-
-        while action is Reduce:
-            rule = arg
-            size = len(rule.expansion)
-            if size:
-                s = value_stack[-size:]
-                del state_stack[-size:]
-                del value_stack[-size:]
-            else:
-                s = []
-
-            value = self.parser.callbacks[rule](s)
-
-            _action, new_state = self.parser.parse_table.states[state_stack[-1]][rule.origin.name]
-            assert _action is Shift
-            state_stack.append(new_state)
-            value_stack.append(value)
-
-            if state_stack[-1] == end_state:
-                return value_stack[-1]
-
-            state = state_stack[-1]
-            action, arg = self.parser.parse_table.states[state][token.type]
-            assert arg != end_state
-
-        assert action is Shift
-        state_stack.append(arg)
-        value_stack.append(token)
-
-
-    def choices(self):
-        return self.parser.parse_table.states[self._state_stack[-1]]
-
-    def resume_parse(self):
-        return self.parser.parse(self._stream, self._start, self._set_state, self._value_stack, self._state_stack)

lark/parsers/lalr_puppet.py

@@ -0,0 +1,76 @@
+# This module provide a LALR puppet, which is used to debugging and error handling
+
+from copy import deepcopy
+
+from .lalr_analysis import Shift, Reduce
+
+class ParserPuppet:
+    def __init__(self, parser, state_stack, value_stack, start, stream, set_state):
+        self.parser = parser
+        self._state_stack = state_stack
+        self._value_stack = value_stack
+        self._start = start
+        self._stream = stream
+        self._set_state = set_state
+
+        self.result = None
+
+    def feed_token(self, token):
+        end_state = self.parser.parse_table.end_states[self._start]
+        state_stack = self._state_stack
+        value_stack = self._value_stack
+
+        state = state_stack[-1]
+        action, arg = self.parser.parse_table.states[state][token.type]
+        assert arg != end_state
+
+        while action is Reduce:
+            rule = arg
+            size = len(rule.expansion)
+            if size:
+                s = value_stack[-size:]
+                del state_stack[-size:]
+                del value_stack[-size:]
+            else:
+                s = []
+
+            value = self.parser.callbacks[rule](s)
+
+            _action, new_state = self.parser.parse_table.states[state_stack[-1]][rule.origin.name]
+            assert _action is Shift
+            state_stack.append(new_state)
+            value_stack.append(value)
+
+            if state_stack[-1] == end_state:
+                self.result = value_stack[-1]
+                return self.result
+
+            state = state_stack[-1]
+            action, arg = self.parser.parse_table.states[state][token.type]
+            assert arg != end_state
+
+        assert action is Shift
+        state_stack.append(arg)
+        value_stack.append(token)
+
+    def copy(self):
+        return type(self)(
+            self.parser,
+            list(self._state_stack),
+            deepcopy(self._value_stack),
+            self._start,
+            self._stream,
+            self._set_state,
+        )
+
+    def pretty():
+        print("Puppet choices:")
+        for k, v in self.choices.items():
+            print('\t-', k, '->', v)
+        print('stack size:', len(self._state_stack))
+
+    def choices(self):
+        return self.parser.parse_table.states[self._state_stack[-1]]
+
+    def resume_parse(self):
+        return self.parser.parse(self._stream, self._start, self._set_state, self._value_stack, self._state_stack)

lark/tree.py

@@ -105,7 +105,7 @@ class Tree(object):
                 stack.append(n)
 
     def __deepcopy__(self, memo):
-        return type(self)(self.data, deepcopy(self.children, memo))
+        return type(self)(self.data, deepcopy(self.children, memo), meta=self._meta)
 
     def copy(self):
         return type(self)(self.data, self.children)

tests/test_tools.py

@@ -50,6 +50,10 @@ class TestStandalone(TestCase):
         x = l.parse('16 candles')
         self.assertEqual(x.children, ['16', 'candles'])
 
+        self.assertRaises(context['UnexpectedToken'], l.parse, 'twelve monkeys')
+        self.assertRaises(context['UnexpectedToken'], l.parse, 'twelve')
+        self.assertRaises(context['UnexpectedCharacters'], l.parse, '$ talks')
+
     def test_contextual(self):
         grammar = """
         start: a b