From e5868415ebc9f9a985549f82ff8137806d78450d Mon Sep 17 00:00:00 2001 From: Mostafa Razavi Date: Sun, 12 May 2019 19:32:50 +0200 Subject: [PATCH 1/3] Implement embedded in-place transformers. See #378. As discussed in issue #378, when an embedded transformer (that is, one passed to the Lark class using the transformer argument), is an inplace transformer (either a subclass of Transformer_InPlace, or with the @v_args(tree=True) decorator), the in-place transformer was not working correctly and in-fact Lark used it like a normal non-in-place transformer, expecting it to return the transformed value. --- lark/parse_tree_builder.py | 12 ++++++++++++ tests/test_parser.py | 27 ++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 977c371..550bc17 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -2,6 +2,7 @@ from .exceptions import GrammarError from .lexer import Token from .tree import Tree from .visitors import InlineTransformer # XXX Deprecated +from .visitors import Transformer_InPlace ###{standalone from functools import partial, wraps @@ -193,6 +194,15 @@ def ptb_inline_args(func): return func(*children) return f +def inplace_transformer(func): + @wraps(func) + def f(children): + # function name in a Transformer is a rule name. + tree = Tree(func.__name__, children) + func(tree) + return tree + return f + class ParseTreeBuilder: def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False, maybe_placeholders=False): self.tree_class = tree_class @@ -231,6 +241,8 @@ class ParseTreeBuilder: # XXX InlineTransformer is deprecated! if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer): f = ptb_inline_args(f) + elif hasattr(f, 'whole_tree') or isinstance(transformer, Transformer_InPlace): + f = inplace_transformer(f) except AttributeError: f = partial(self.tree_class, user_callback_name) diff --git a/tests/test_parser.py b/tests/test_parser.py index ce8b7d6..0fddf14 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -20,7 +20,7 @@ logging.basicConfig(level=logging.INFO) from lark.lark import Lark from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters from lark.tree import Tree -from lark.visitors import Transformer +from lark.visitors import Transformer, Transformer_InPlace, v_args from lark.grammar import Rule from lark.lexer import TerminalDef @@ -150,6 +150,31 @@ class TestParsers(unittest.TestCase): r = g.parse("xx") self.assertEqual( r.children, [""] ) + def test_embedded_transformer_inplace(self): + class T1(Transformer_InPlace): + def a(self, tree): + assert isinstance(tree, Tree) + tree.children.append("tested") + + @v_args(tree=True) + class T2(Transformer): + def a(self, tree): + assert isinstance(tree, Tree) + tree.children.append("tested") + + class T3(Transformer): + @v_args(tree=True) + def a(self, tree): + assert isinstance(tree, Tree) + tree.children.append("tested") + + for t in [T1(), T2(), T3()]: + g = Lark("""start: a + a : "x" + """, parser='lalr', transformer=t) + r = g.parse("x") + first, = r.children + self.assertEqual(first.children, ["tested"]) def test_alias(self): Lark("""start: ["a"] "b" ["c"] "e" ["f"] ["g"] ["h"] "x" -> d """) From f71df240b65c8425b6b10b4beb60fdee92f74cf6 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sun, 12 May 2019 21:34:21 +0300 Subject: [PATCH 2/3] Removed Python2 incompatibility --- lark/parser_frontends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index ab69d01..0634814 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -107,7 +107,7 @@ class LALR_ContextualLexer(LALR_WithLexer): ###} class LALR_CustomLexer(LALR_WithLexer): - def __init__(self, lexer_cls, lexer_conf, parser_conf, *, options=None): + def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None): self.lexer = lexer_cls(self.lexer_conf) debug = options.debug if options else False self.parser = LALR_Parser(parser_conf, debug=debug) From a9106df824133f748e33544c6128cc355bc03dab Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sun, 12 May 2019 22:11:13 +0300 Subject: [PATCH 3/3] Corrected thee Transformer's whole_tree interface, for both internal and external use --- lark/parse_tree_builder.py | 3 +-- lark/visitors.py | 2 +- tests/test_parser.py | 34 +++++++++++++++++++++++++++------- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 550bc17..b54b6e8 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -199,8 +199,7 @@ def inplace_transformer(func): def f(children): # function name in a Transformer is a rule name. tree = Tree(func.__name__, children) - func(tree) - return tree + return func(tree) return f class ParseTreeBuilder: diff --git a/lark/visitors.py b/lark/visitors.py index 53847f9..4a0f639 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -36,7 +36,7 @@ class Transformer: return f(*children) elif getattr(f, 'whole_tree', False): if new_children is not None: - raise NotImplementedError("Doesn't work with the base Transformer class") + tree.children = new_children return f(tree) else: return f(children) diff --git a/tests/test_parser.py b/tests/test_parser.py index 0fddf14..1cf702d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -151,30 +151,50 @@ class TestParsers(unittest.TestCase): self.assertEqual( r.children, [""] ) def test_embedded_transformer_inplace(self): + @v_args(tree=True) class T1(Transformer_InPlace): def a(self, tree): - assert isinstance(tree, Tree) + assert isinstance(tree, Tree), tree tree.children.append("tested") + return tree + + def b(self, tree): + return Tree(tree.data, tree.children + ['tested2']) @v_args(tree=True) class T2(Transformer): def a(self, tree): assert isinstance(tree, Tree) tree.children.append("tested") + return tree + + def b(self, tree): + return Tree(tree.data, tree.children + ['tested2']) class T3(Transformer): @v_args(tree=True) def a(self, tree): assert isinstance(tree, Tree) tree.children.append("tested") + return tree + + @v_args(tree=True) + def b(self, tree): + return Tree(tree.data, tree.children + ['tested2']) for t in [T1(), T2(), T3()]: - g = Lark("""start: a - a : "x" - """, parser='lalr', transformer=t) - r = g.parse("x") - first, = r.children - self.assertEqual(first.children, ["tested"]) + for internal in [False, True]: + g = Lark("""start: a b + a : "x" + b : "y" + """, parser='lalr', transformer=t if internal else None) + r = g.parse("xy") + if not internal: + r = t.transform(r) + + a, b = r.children + self.assertEqual(a.children, ["tested"]) + self.assertEqual(b.children, ["tested2"]) def test_alias(self): Lark("""start: ["a"] "b" ["c"] "e" ["f"] ["g"] ["h"] "x" -> d """)