From 6bfc27c11d2d17cc6ea9ee6fab77ff75806b9d87 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Fri, 18 May 2018 15:14:57 +0300 Subject: [PATCH] New transformers near completion Nearley tool still needs fixing --- examples/calc.py | 5 +- examples/json_parser.py | 6 +- lark/__init__.py | 3 +- lark/load_grammar.py | 15 ++-- lark/parse_tree_builder.py | 16 +++- lark/parsers/earley.py | 11 +-- lark/parsers/resolve_ambig.py | 11 ++- lark/tree.py | 4 +- lark/visitors.py | 145 ++++++++++++++++++++-------------- tests/test_parser.py | 3 +- tests/test_trees.py | 30 ++----- 11 files changed, 140 insertions(+), 109 deletions(-) diff --git a/examples/calc.py b/examples/calc.py index e90b5cc..1102151 100644 --- a/examples/calc.py +++ b/examples/calc.py @@ -2,7 +2,7 @@ # This example shows how to write a basic calculator with variables. # -from lark import Lark, Transformer, children_args_inline +from lark import Lark, Transformer, visitor_args try: input = raw_input # For Python2 compatibility @@ -34,7 +34,8 @@ calc_grammar = """ %ignore WS_INLINE """ -class CalculateTree(SimpleTransformer): +@visitor_args(inline=True) +class CalculateTree(Transformer): from operator import add, sub, mul, truediv as div, neg number = float diff --git a/examples/json_parser.py b/examples/json_parser.py index 38a5b70..216af2c 100644 --- a/examples/json_parser.py +++ b/examples/json_parser.py @@ -7,7 +7,7 @@ import sys -from lark import Lark, inline_args, Transformer +from lark import Lark, Transformer, visitor_args json_grammar = r""" ?start: value @@ -34,14 +34,14 @@ json_grammar = r""" """ class TreeToJson(Transformer): - @inline_args + @visitor_args(inline=True) def string(self, s): return s[1:-1].replace('\\"', '"') array = list pair = tuple object = dict - number = inline_args(float) + number = visitor_args(inline=True)(float) null = lambda self, _: None true = lambda self, _: True diff --git a/lark/__init__.py b/lark/__init__.py index b36b3fc..850f702 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,5 +1,6 @@ from .tree import Tree -from .visitors import Transformer, Visitor, children_args, children_args_inline +from .visitors import Transformer, Visitor, visitor_args, Discard +from .visitors import InlineTransformer, inline_args # XXX Deprecated from .common import ParseError, GrammarError, UnexpectedToken from .lexer import UnexpectedInput, LexError from .lark import Lark diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 6262f62..5cf8e19 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -16,7 +16,7 @@ from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol from .utils import classify, suppress from .tree import Tree, SlottedTree as ST -from .visitors import Transformer, Visitor, children_args, children_args_inline +from .visitors import Transformer, Visitor, visitor_args __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -138,7 +138,7 @@ RULES = { } -@children_args_inline +@visitor_args(inline=True) class EBNF_to_BNF(Transformer): def __init__(self): self.new_rules = [] @@ -232,7 +232,6 @@ class SimplifyRule_Visitor(Visitor): tree.children = list(set(tree.children)) -@children_args class RuleTreeToText(Transformer): def expansions(self, x): return x @@ -244,7 +243,7 @@ class RuleTreeToText(Transformer): return expansion, alias.value -@children_args_inline +@visitor_args(inline=True) class CanonizeTree(Transformer): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) @@ -265,7 +264,7 @@ class PrepareAnonTerminals(Transformer): self.i = 0 - @children_args_inline + @visitor_args(inline=True) def pattern(self, p): value = p.value if p in self.token_reverse and p.flags != self.token_reverse[p].pattern.flags: @@ -355,7 +354,7 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -@children_args_inline +@visitor_args(inline=True) class PrepareLiterals(Transformer): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -369,7 +368,6 @@ class PrepareLiterals(Transformer): return ST('pattern', [PatternRE(regexp)]) -@children_args class TokenTreeToPattern(Transformer): def pattern(self, ps): p ,= ps @@ -410,7 +408,6 @@ class TokenTreeToPattern(Transformer): return v[0] class PrepareSymbols(Transformer): - @children_args def value(self, v): v ,= v if isinstance(v, Tree): @@ -535,7 +532,7 @@ def options_from_rule(name, *x): def symbols_from_strcase(expansion): return [Terminal(x, filter_out=x.startswith('_')) if is_terminal(x) else NonTerminal(x) for x in expansion] -@children_args_inline +@visitor_args(inline=True) class PrepareGrammar(Transformer): def terminal(self, name): return name diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 7c624e2..abca756 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -3,9 +3,10 @@ from .utils import suppress from .lexer import Token from .grammar import Rule from .tree import Tree +from .visitors import InlineTransformer # XXX Deprecated ###{standalone -from functools import partial +from functools import partial, wraps class ExpandSingleChild: @@ -95,6 +96,15 @@ def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous): class Callback(object): pass + +def inline_args(func): + @wraps(func) + def f(children): + return func(*children) + return f + + + class ParseTreeBuilder: def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False): self.tree_class = tree_class @@ -130,6 +140,10 @@ class ParseTreeBuilder: user_callback_name = rule.alias or rule.origin.name try: f = getattr(transformer, user_callback_name) + assert not getattr(f, 'meta', False), "Meta args not supported for internal transformer" + # XXX InlineTransformer is deprecated! + if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer): + f = inline_args(f) except AttributeError: f = partial(self.tree_class, user_callback_name) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 87dc41e..46a1271 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,7 @@ # Email : erezshin@gmail.com from ..tree import Tree -from ..visitors import Transformer_InPlace +from ..visitors import Transformer_InPlace, visitor_args from ..common import ParseError, UnexpectedToken from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal @@ -114,9 +114,9 @@ class Column: if old_tree.data != '_ambig': new_tree = old_tree.copy() - new_tree.rule = old_tree.rule + new_tree.meta.rule = old_tree.meta.rule old_tree.set('_ambig', [new_tree]) - old_tree.rule = None # No longer a 'drv' node + old_tree.meta.rule = None # No longer a 'drv' node if item.tree.children[0] is old_tree: # XXX a little hacky! raise ParseError("Infinite recursion in grammar! (Rule %s)" % item.rule) @@ -234,5 +234,6 @@ class ApplyCallbacks(Transformer_InPlace): def __init__(self, postprocess): self.postprocess = postprocess - def drv(self, tree): - return self.postprocess[tree.meta.rule](tree.children) + @visitor_args(meta=True) + def drv(self, children, meta): + return self.postprocess[meta.rule](children) diff --git a/lark/parsers/resolve_ambig.py b/lark/parsers/resolve_ambig.py index 0d3f17c..2470eb9 100644 --- a/lark/parsers/resolve_ambig.py +++ b/lark/parsers/resolve_ambig.py @@ -26,8 +26,15 @@ def _compare_priority(tree1, tree2): tree1.iter_subtrees() def _compare_drv(tree1, tree2): - rule1 = getattr(tree1.meta, 'rule', None) - rule2 = getattr(tree2.meta, 'rule', None) + try: + rule1 = tree1.meta.rule + except AttributeError: + rule1 = None + + try: + rule2 = tree2.meta.rule + except AttributeError: + rule2 = None if None == rule1 == rule2: return compare(tree1, tree2) diff --git a/lark/tree.py b/lark/tree.py index e20c18d..1490632 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -10,10 +10,10 @@ class Meta: ###{standalone class Tree(object): - def __init__(self, data, children): + def __init__(self, data, children, meta=None): self.data = data self.children = children - self._meta = None + self._meta = meta @property def meta(self): diff --git a/lark/visitors.py b/lark/visitors.py index d3853bf..950371f 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -8,39 +8,23 @@ class Discard(Exception): pass -class Base: - def _call_userfunc(self, tree): - return getattr(self, tree.data, self.__default__)(tree) +# Transformers - def __default__(self, tree): - "Default operation on tree (for override)" - return tree - - @classmethod - def _apply_decorator(cls, decorator): - mro = getmro(cls) - assert mro[0] is cls - libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)} - for name, value in getmembers(cls): - if name.startswith('_') or name in libmembers: - continue - - setattr(cls, name, decorator(value)) - return cls - - -class SimpleBase(Base): - def _call_userfunc(self, tree): +class Transformer: + def _call_userfunc(self, data, children, meta): # Assumes tree is already transformed try: - f = getattr(self, tree.data) + f = getattr(self, data) except AttributeError: - return self.__default__(tree) + return self.__default__(data, children, meta) else: - return f(tree.children) - + if getattr(f, 'meta', False): + return f(children, meta) + elif getattr(f, 'inline', False): + return f(*children) + else: + return f(children) -class Transformer(Base): def _transform_children(self, children): for c in children: try: @@ -49,8 +33,8 @@ class Transformer(Base): pass def _transform_tree(self, tree): - tree = Tree(tree.data, list(self._transform_children(tree.children))) - return self._call_userfunc(tree) + children = list(self._transform_children(tree.children)) + return self._call_userfunc(tree.data, children, tree.meta) def transform(self, tree): return self._transform_tree(tree) @@ -58,6 +42,32 @@ class Transformer(Base): def __mul__(self, other): return TransformerChain(self, other) + def __default__(self, data, children, meta): + "Default operation on tree (for override)" + return Tree(data, children, meta) + + @classmethod + def _apply_decorator(cls, decorator, **kwargs): + mro = getmro(cls) + assert mro[0] is cls + libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)} + for name, value in getmembers(cls): + if name.startswith('_') or name in libmembers: + continue + + setattr(cls, name, decorator(value, **kwargs)) + return cls + + +class InlineTransformer(Transformer): # XXX Deprecated + def _call_userfunc(self, data, children, meta): + # Assumes tree is already transformed + try: + f = getattr(self, data) + except AttributeError: + return self.__default__(data, children, meta) + else: + return f(*children) class TransformerChain(object): @@ -75,7 +85,7 @@ class TransformerChain(object): class Transformer_InPlace(Transformer): def _transform_tree(self, tree): # Cancel recursion - return self._call_userfunc(tree) + return self._call_userfunc(tree.data, tree.children, tree.meta) def transform(self, tree): for subtree in tree.iter_subtrees(): @@ -87,11 +97,22 @@ class Transformer_InPlace(Transformer): class Transformer_InPlaceRecursive(Transformer): def _transform_tree(self, tree): tree.children = list(self._transform_children(tree.children)) - return self._call_userfunc(tree) + return self._call_userfunc(tree.data, tree.children, tree.meta) + +# Visitors -class Visitor(Base): +class VisitorBase: + def _call_userfunc(self, tree): + return getattr(self, tree.data, self.__default__)(tree) + + def __default__(self, tree): + "Default operation on tree (for override)" + return tree + + +class Visitor(VisitorBase): "Bottom-up visitor" def visit(self, tree): @@ -99,7 +120,7 @@ class Visitor(Base): self._call_userfunc(subtree) return tree -class Visitor_Recursive(Base): +class Visitor_Recursive(VisitorBase): def visit(self, tree): for child in tree.children: if isinstance(child, Tree): @@ -110,6 +131,7 @@ class Visitor_Recursive(Base): return tree + def visit_children_decor(func): @wraps(func) def inner(cls, tree): @@ -117,7 +139,8 @@ def visit_children_decor(func): return func(cls, values) return inner -class Interpreter(object): + +class Interpreter: "Top-down visitor" def visit(self, tree): @@ -136,56 +159,58 @@ class Interpreter(object): +# Decorators -def _apply_decorator(obj, decorator): +def _apply_decorator(obj, decorator, **kwargs): try: _apply = obj._apply_decorator except AttributeError: - return decorator(obj) + return decorator(obj, **kwargs) else: - return _apply(decorator) + return _apply(decorator, **kwargs) -def _children_args__func(func): - if getattr(func, '_children_args_decorated', False): - return func +def _inline_args__func(func): @wraps(func) def create_decorator(_f, with_self): if with_self: - def f(self, tree): - return _f(self, tree.children) + def f(self, children): + return _f(self, *children) else: - def f(args): - return _f(tree.children) - f._children_args_decorated = True + def f(self, children): + return _f(*children) return f return smart_decorator(func, create_decorator) -def children_args(obj): - return _apply_decorator(obj, _children_args__func) +def inline_args(obj): # XXX Deprecated + return _apply_decorator(obj, _inline_args__func) -def _children_args_inline__func(func): - if getattr(func, '_children_args_decorated', False): - return func - @wraps(func) +def _visitor_args_func_dec(func, inline=False, meta=False): + assert not (inline and meta) def create_decorator(_f, with_self): if with_self: - def f(self, tree): - return _f(self, *tree.children) + def f(self, *args, **kwargs): + return _f(self, *args, **kwargs) else: - def f(self, tree): - print ('##', _f, tree) - return _f(*tree.children) - f._children_args_decorated = True + def f(self, *args, **kwargs): + return _f(*args, **kwargs) return f - return smart_decorator(func, create_decorator) + f = smart_decorator(func, create_decorator) + f.inline = inline + f.meta = meta + return f + +def visitor_args(inline=False, meta=False): + if inline and meta: + raise ValueError("Visitor functions can either accept meta, or be inlined. Not both.") + def _visitor_args_dec(obj): + return _apply_decorator(obj, _visitor_args_func_dec, inline=inline, meta=meta) + return _visitor_args_dec -def children_args_inline(obj): - return _apply_decorator(obj, _children_args_inline__func) diff --git a/tests/test_parser.py b/tests/test_parser.py index 25ce619..f48f3bd 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,7 +21,7 @@ from lark.lark import Lark from lark.common import GrammarError, ParseError, UnexpectedToken from lark.lexer import LexError, UnexpectedInput from lark.tree import Tree -from lark.visitors import Transformer, children_args +from lark.visitors import Transformer __path__ = os.path.dirname(__file__) def _read(n, *args): @@ -93,7 +93,6 @@ class TestParsers(unittest.TestCase): self.assertEqual( r.children[0].data, "c" ) def test_embedded_transformer(self): - @children_args class T(Transformer): def a(self, children): return "" diff --git a/tests/test_trees.py b/tests/test_trees.py index b7796bf..af5a2a0 100644 --- a/tests/test_trees.py +++ b/tests/test_trees.py @@ -6,7 +6,7 @@ import copy import pickle from lark.tree import Tree -from lark.visitors import Transformer, Interpreter, visit_children_decor, children_args_inline, children_args +from lark.visitors import Transformer, Interpreter, visit_children_decor, visitor_args class TestTrees(TestCase): @@ -63,19 +63,18 @@ class TestTrees(TestCase): t = Tree('add', [Tree('sub', [Tree('i', ['3']), Tree('f', ['1.1'])]), Tree('i', ['1'])]) class T(Transformer): - i = children_args_inline(int) - f = children_args_inline(float) + i = visitor_args(inline=True)(int) + f = visitor_args(inline=True)(float) - sub = lambda self, tree: tree.children[0] - tree.children[1] - - def add(self, tree): - return sum(tree.children) + sub = lambda self, values: values[0] - values[1] + def add(self, values): + return sum(values) res = T().transform(t) self.assertEqual(res, 2.9) - @children_args_inline + @visitor_args(inline=True) class T(Transformer): i = int f = float @@ -89,7 +88,7 @@ class TestTrees(TestCase): self.assertEqual(res, 2.9) - @children_args_inline + @visitor_args(inline=True) class T(Transformer): i = int f = float @@ -99,19 +98,6 @@ class TestTrees(TestCase): self.assertEqual(res, 2.9) - @children_args - class T(Transformer): - i = children_args_inline(int) - f = children_args_inline(float) - - sub = lambda self, values: values[0] - values[1] - - def add(self, values): - return sum(values) - - res = T().transform(t) - self.assertEqual(res, 2.9) - if __name__ == '__main__': unittest.main()