From cc1092bd5339a1d41842f1eddfc138f0932f529e Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Fri, 26 Jun 2020 16:42:01 +0300 Subject: [PATCH] Refactored puppet + small fixes --- lark/parsers/lalr_parser.py | 64 ++++--------------------------- lark/parsers/lalr_puppet.py | 76 +++++++++++++++++++++++++++++++++++++ lark/tree.py | 2 +- tests/test_tools.py | 4 ++ 4 files changed, 88 insertions(+), 58 deletions(-) create mode 100644 lark/parsers/lalr_puppet.py diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py index 7d5cf3b..f26cbc5 100644 --- a/lark/parsers/lalr_parser.py +++ b/lark/parsers/lalr_parser.py @@ -7,9 +7,10 @@ from ..lexer import Token from ..utils import Enumerator, Serialize from .lalr_analysis import LALR_Analyzer, Shift, Reduce, IntParseTable - +from .lalr_puppet import ParserPuppet ###{standalone + class LALR_Parser(object): def __init__(self, parser_conf, debug=False): assert all(r.options.priority is None for r in parser_conf.rules), "LALR doesn't yet support prioritization" @@ -59,7 +60,11 @@ class _Parser: return states[state][token.type] except KeyError: expected = [s for s in states[state].keys() if s.isupper()] - raise UnexpectedToken(token, expected, state=state, puppet=_ParserPuppet(self, state_stack, value_stack, start, stream, set_state)) + try: + puppet = ParserPuppet(self, state_stack, value_stack, start, stream, set_state) + except NameError: + puppet = None + raise UnexpectedToken(token, expected, state=state, puppet=puppet) def reduce(rule): size = len(rule.expansion) @@ -112,58 +117,3 @@ class _Parser: ###} - - - -class _ParserPuppet: - def __init__(self, parser, state_stack, value_stack, start, stream, set_state): - self.parser = parser - self._state_stack = state_stack - self._value_stack = value_stack - self._start = start - self._stream = stream - self._set_state = set_state - - def feed_token(self, token): - end_state = self.parser.parse_table.end_states[self._start] - state_stack = self._state_stack - value_stack = self._value_stack - - state = state_stack[-1] - action, arg = self.parser.parse_table.states[state][token.type] - assert arg != end_state - - while action is Reduce: - rule = arg - size = len(rule.expansion) - if size: - s = value_stack[-size:] - del state_stack[-size:] - del value_stack[-size:] - else: - s = [] - - value = self.parser.callbacks[rule](s) - - _action, new_state = self.parser.parse_table.states[state_stack[-1]][rule.origin.name] - assert _action is Shift - state_stack.append(new_state) - value_stack.append(value) - - if state_stack[-1] == end_state: - return value_stack[-1] - - state = state_stack[-1] - action, arg = self.parser.parse_table.states[state][token.type] - assert arg != end_state - - assert action is Shift - state_stack.append(arg) - value_stack.append(token) - - - def choices(self): - return self.parser.parse_table.states[self._state_stack[-1]] - - def resume_parse(self): - return self.parser.parse(self._stream, self._start, self._set_state, self._value_stack, self._state_stack) \ No newline at end of file diff --git a/lark/parsers/lalr_puppet.py b/lark/parsers/lalr_puppet.py new file mode 100644 index 0000000..14e21fc --- /dev/null +++ b/lark/parsers/lalr_puppet.py @@ -0,0 +1,76 @@ +# This module provide a LALR puppet, which is used to debugging and error handling + +from copy import deepcopy + +from .lalr_analysis import Shift, Reduce + +class ParserPuppet: + def __init__(self, parser, state_stack, value_stack, start, stream, set_state): + self.parser = parser + self._state_stack = state_stack + self._value_stack = value_stack + self._start = start + self._stream = stream + self._set_state = set_state + + self.result = None + + def feed_token(self, token): + end_state = self.parser.parse_table.end_states[self._start] + state_stack = self._state_stack + value_stack = self._value_stack + + state = state_stack[-1] + action, arg = self.parser.parse_table.states[state][token.type] + assert arg != end_state + + while action is Reduce: + rule = arg + size = len(rule.expansion) + if size: + s = value_stack[-size:] + del state_stack[-size:] + del value_stack[-size:] + else: + s = [] + + value = self.parser.callbacks[rule](s) + + _action, new_state = self.parser.parse_table.states[state_stack[-1]][rule.origin.name] + assert _action is Shift + state_stack.append(new_state) + value_stack.append(value) + + if state_stack[-1] == end_state: + self.result = value_stack[-1] + return self.result + + state = state_stack[-1] + action, arg = self.parser.parse_table.states[state][token.type] + assert arg != end_state + + assert action is Shift + state_stack.append(arg) + value_stack.append(token) + + def copy(self): + return type(self)( + self.parser, + list(self._state_stack), + deepcopy(self._value_stack), + self._start, + self._stream, + self._set_state, + ) + + def pretty(): + print("Puppet choices:") + for k, v in self.choices.items(): + print('\t-', k, '->', v) + print('stack size:', len(self._state_stack)) + + def choices(self): + return self.parser.parse_table.states[self._state_stack[-1]] + + def resume_parse(self): + return self.parser.parse(self._stream, self._start, self._set_state, self._value_stack, self._state_stack) diff --git a/lark/tree.py b/lark/tree.py index 84c996a..f9767e4 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -105,7 +105,7 @@ class Tree(object): stack.append(n) def __deepcopy__(self, memo): - return type(self)(self.data, deepcopy(self.children, memo)) + return type(self)(self.data, deepcopy(self.children, memo), meta=self._meta) def copy(self): return type(self)(self.data, self.children) diff --git a/tests/test_tools.py b/tests/test_tools.py index 5316396..1e0d78e 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -50,6 +50,10 @@ class TestStandalone(TestCase): x = l.parse('16 candles') self.assertEqual(x.children, ['16', 'candles']) + self.assertRaises(context['UnexpectedToken'], l.parse, 'twelve monkeys') + self.assertRaises(context['UnexpectedToken'], l.parse, 'twelve') + self.assertRaises(context['UnexpectedCharacters'], l.parse, '$ talks') + def test_contextual(self): grammar = """ start: a b