diff --git a/lark/exceptions.py b/lark/exceptions.py index 28f1b4b..f46fa82 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -97,10 +97,10 @@ class UnexpectedToken(ParseError, UnexpectedInput): super(UnexpectedToken, self).__init__(message) class VisitError(LarkError): - def __init__(self, tree, orig_exc): - self.tree = tree + def __init__(self, rule, obj, orig_exc): + self.obj = obj self.orig_exc = orig_exc - message = 'Error trying to process rule "%s":\n\n%s' % (tree.data, orig_exc) + message = 'Error trying to process rule "%s":\n\n%s' % (rule, orig_exc) super(VisitError, self).__init__(message) ###} diff --git a/lark/lark.py b/lark/lark.py index db1dfd2..d334cc7 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -225,7 +225,16 @@ class Lark(Serialize): for rule in self.rules: if rule.options and rule.options.priority is not None: rule.options.priority = None - self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, self.options.lexer_callbacks) + + # TODO Deprecate lexer_callbacks? + lexer_callbacks = dict(self.options.lexer_callbacks) + if self.options.transformer: + t = self.options.transformer + for term in self.terminals: + if hasattr(t, term.name): + lexer_callbacks[term.name] = getattr(t, term.name) + + self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, lexer_callbacks) if self.options.parser: self.parser = self._build_parser() diff --git a/lark/lexer.py b/lark/lexer.py index 806d575..21951e4 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -108,6 +108,13 @@ class Token(Str): self.end_column = end_column return self + def update(self, type_=None, value=None): + return Token.new_borrow_pos( + type_ if type_ is not None else self.type, + value if value is not None else self.value, + self + ) + @classmethod def new_borrow_pos(cls, type_, value, borrow_t): return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column) diff --git a/lark/visitors.py b/lark/visitors.py index 4f32091..a2d5e86 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -48,7 +48,7 @@ class Transformer: except (GrammarError, Discard): raise except Exception as e: - raise VisitError(tree, e) + raise VisitError(tree.data, tree, e) def _call_userfunc_token(self, token): try: @@ -61,7 +61,7 @@ class Transformer: except (GrammarError, Discard): raise except Exception as e: - raise VisitError(token, e) + raise VisitError(token.type, token, e) def _transform_children(self, children): diff --git a/tests/test_parser.py b/tests/test_parser.py index 35b3015..caee80e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -99,16 +99,22 @@ class TestParsers(unittest.TestCase): def a(self, children): return children[0] + "!" def A(self, tok): - return tok.upper() + return tok.update(value=tok.upper()) # Test regular - g = Lark("""start: a - a : A - A: "x" - """, parser='lalr') - r = T().transform(g.parse("x")) + g = """start: a + a : A + A: "x" + """ + p = Lark(g, parser='lalr') + r = T(False).transform(p.parse("x")) self.assertEqual( r.children, ["x!"] ) - r = T(True).transform(g.parse("x")) + r = T().transform(p.parse("x")) + self.assertEqual( r.children, ["X!"] ) + + # Test internal transformer + p = Lark(g, parser='lalr', transformer=T()) + r = p.parse("x") self.assertEqual( r.children, ["X!"] )