Refactored transformers, better code

lark/__init__.py

@@ -3,6 +3,5 @@ from .transformers import Transformer
 from .common import ParseError, GrammarError, UnexpectedToken
 from .lexer import UnexpectedInput, LexError
 from .lark import Lark
-from .utils import inline_args
 
 __version__ = "0.5.6"

lark/load_grammar.py

@@ -16,7 +16,7 @@ from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
 from .utils import classify, suppress
 
 from .tree import Tree, SlottedTree as ST
-from .transformers import Transformer, ChildrenTransformer, inline_args, Visitor
+from .transformers import Transformer, Visitor, children_args, children_args_inline
 
 __path__ = os.path.dirname(__file__)
 IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
@@ -138,8 +138,8 @@ RULES = {
 }
 
 
-@inline_args
-class EBNF_to_BNF(ChildrenTransformer):
+@children_args_inline
+class EBNF_to_BNF(Transformer):
     def __init__(self):
         self.new_rules = []
         self.rules_by_expr = {}
@@ -232,7 +232,8 @@ class SimplifyRule_Visitor(Visitor):
         tree.children = list(set(tree.children))
 
 
-class RuleTreeToText(ChildrenTransformer):
+@children_args
+class RuleTreeToText(Transformer):
     def expansions(self, x):
         return x
     def expansion(self, symbols):
@@ -243,8 +244,8 @@ class RuleTreeToText(ChildrenTransformer):
         return expansion, alias.value
 
 
-@inline_args
-class CanonizeTree(ChildrenTransformer):
+@children_args_inline
+class CanonizeTree(Transformer):
     def maybe(self, expr):
         return ST('expr', [expr, Token('OP', '?', -1)])
 
@@ -254,8 +255,8 @@ class CanonizeTree(ChildrenTransformer):
         tokenmods, value = args
         return tokenmods + [value]
 
-@inline_args
-class PrepareAnonTerminals(ChildrenTransformer):
+@children_args_inline
+class PrepareAnonTerminals(Transformer):
     "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them"
 
     def __init__(self, tokens):
@@ -354,8 +355,8 @@ def _literal_to_pattern(literal):
              'REGEXP': PatternRE }[literal.type](s, flags)
 
 
-@inline_args
-class PrepareLiterals(ChildrenTransformer):
+@children_args_inline
+class PrepareLiterals(Transformer):
     def literal(self, literal):
         return ST('pattern', [_literal_to_pattern(literal)])
 
@@ -368,7 +369,8 @@ class PrepareLiterals(ChildrenTransformer):
         return ST('pattern', [PatternRE(regexp)])
 
 
-class TokenTreeToPattern(ChildrenTransformer):
+@children_args
+class TokenTreeToPattern(Transformer):
     def pattern(self, ps):
         p ,= ps
         return p
@@ -407,7 +409,8 @@ class TokenTreeToPattern(ChildrenTransformer):
     def value(self, v):
         return v[0]
 
-class PrepareSymbols(ChildrenTransformer):
+@children_args
+class PrepareSymbols(Transformer):
     def value(self, v):
         v ,= v
         if isinstance(v, Tree):
@@ -532,8 +535,8 @@ def options_from_rule(name, *x):
 def symbols_from_strcase(expansion):
     return [Terminal(x, filter_out=x.startswith('_')) if is_terminal(x) else NonTerminal(x) for x in expansion]
 
-@inline_args
-class PrepareGrammar(ChildrenTransformer):
+@children_args_inline
+class PrepareGrammar(Transformer):
     def terminal(self, name):
         return name
     def nonterminal(self, name):

lark/transformers.py

@@ -1,7 +1,7 @@
 import inspect
 from functools import wraps
 
-from . import utils
+from .utils import smart_decorator
 from .tree import Tree
 
 class Discard(Exception):
@@ -13,46 +13,27 @@ class Base:
         return getattr(self, tree.data, self.__default__)(tree)
 
     def __default__(self, tree):
+        "Default operation on tree (for override)"
         return tree
 
 class Transformer(Base):
     def _transform_children(self, children):
         for c in children:
             try:
-                yield self._transform(c) if isinstance(c, Tree) else c
+                yield self._transform_tree(c) if isinstance(c, Tree) else c
             except Discard:
                 pass
 
-    def _transform(self, tree):
+    def _transform_tree(self, tree):
         tree = Tree(tree.data, list(self._transform_children(tree.children)))
         return self._call_userfunc(tree)
 
     def transform(self, tree):
-        return self._transform(tree)
+        return self._transform_tree(tree)
 
     def __mul__(self, other):
         return TransformerChain(self, other)
 
-class ChildrenTransformer(Transformer):
-    def _call_userfunc(self, tree):
-        # Assumes tree is already transformed
-        try:
-            f = getattr(self, tree.data)
-        except AttributeError:
-            return self.__default__(tree)
-        else:
-            return f(tree.children)
-
-class ChildrenInlineTransformer(Transformer):
-    def _call_userfunc(self, tree):
-        # Assumes tree is already transformed
-        try:
-            f = getattr(self, tree.data)
-        except AttributeError:
-            return self.__default__(tree)
-        else:
-            return f(*tree.children)
-
 
 class TransformerChain(object):
     def __init__(self, *transformers):
@@ -68,14 +49,22 @@ class TransformerChain(object):
 
 
 class Transformer_InPlace(Transformer):
-    def _transform(self, tree):
+    def _transform_tree(self, tree):           # Cancel recursion
         return self._call_userfunc(tree)
 
     def transform(self, tree):
         for subtree in tree.iter_subtrees():
             subtree.children = list(self._transform_children(subtree.children))
 
-        return self._transform(tree)
+        return self._transform_tree(tree)
+
+
+class Transformer_InPlaceRecursive(Transformer):
+    def _transform_tree(self, tree):
+        tree.children = list(self._transform_children(tree.children))
+        return self._call_userfunc(tree)
+
+
 
 class Visitor(Base):
     "Bottom-up visitor"
@@ -85,11 +74,6 @@ class Visitor(Base):
             self._call_userfunc(subtree)
         return tree
 
-class Transformer_InPlaceRecursive(Transformer):
-    def _transform(self, tree):
-        tree.children = list(self._transform_children(tree.children))
-        return self._call_userfunc(tree)
-
 class Visitor_Recursive(Base):
     def visit(self, tree):
         for child in tree.children:
@@ -101,7 +85,6 @@ class Visitor_Recursive(Base):
         return tree
 
 
-from functools import wraps
 def visit_children_decor(func):
     @wraps(func)
     def inner(cls, tree):
@@ -126,11 +109,63 @@ class Interpreter(object):
         return self.visit_children(tree)
 
 
-def inline_args(obj):
-    if inspect.isclass(obj) and issubclass(obj, ChildrenTransformer):
-        class _NewTransformer(ChildrenInlineTransformer, obj):
-            pass
-        return _NewTransformer
-    else:
-        return utils.inline_args(obj)
 
+def _children_args__func(f):
+    @wraps(f)
+    def create_decorator(_f, with_self):
+        if with_self:
+            def f(self, tree):
+                return _f(self, tree.children)
+        else:
+            def f(args):
+                return _f(tree.children)
+
+    return smart_decorator(f, create_decorator)
+
+def _children_args__class(cls):
+    def _call_userfunc(self, tree):
+        # Assumes tree is already transformed
+        try:
+            f = getattr(self, tree.data)
+        except AttributeError:
+            return self.__default__(tree)
+        else:
+            return f(tree.children)
+    cls._call_userfunc = _call_userfunc
+    return cls
+
+
+def children_args(obj):
+    decorator = _children_args__class if issubclass(obj, Base) else _children_args__func
+    return decorator(obj)
+
+
+
+def _children_args_inline__func(f):
+    @wraps(f)
+    def create_decorator(_f, with_self):
+        if with_self:
+            def f(self, tree):
+                return _f(self, *tree.children)
+        else:
+            def f(args):
+                return _f(*tree.children)
+
+    return smart_decorator(f, create_decorator)
+
+
+def _children_args_inline__class(cls):
+    def _call_userfunc(self, tree):
+        # Assumes tree is already transformed
+        try:
+            f = getattr(self, tree.data)
+        except AttributeError:
+            return self.__default__(tree)
+        else:
+            return f(*tree.children)
+    cls._call_userfunc = _call_userfunc
+    return cls
+
+def children_args_inline(obj):
+    decorator = _children_args_inline__class if issubclass(obj, Base) else _children_args_inline__func
+    return decorator(obj)

lark/tree.py

@@ -5,8 +5,6 @@ except ImportError:
 
 from copy import deepcopy
 
-from .utils import inline_args
-
 class Meta:
     pass
 

lark/utils.py

@@ -50,22 +50,22 @@ except NameError:   # Python 3
 ###{standalone
 
 import types
-import functools
+from functools import wraps, partial
 from contextlib import contextmanager
 
 Str = type(u'')
 
 def smart_decorator(f, create_decorator):
     if isinstance(f, types.FunctionType):
-        return functools.wraps(create_decorator(f, True))
+        return wraps(create_decorator(f, True))
 
     elif isinstance(f, (type, types.BuiltinFunctionType)):
-        return functools.wraps(create_decorator(f, False))
+        return wraps(create_decorator(f, False))
 
     elif isinstance(f, types.MethodType):
-        return functools.wraps(create_decorator(f.__func__, True))
+        return wraps(create_decorator(f.__func__, True))
 
-    elif isinstance(f, functools.partial):
+    elif isinstance(f, partial):
         # wraps does not work for partials in 2.7: https://bugs.python.org/issue3445
         return create_decorator(f.__func__, True)
 
@@ -73,16 +73,6 @@ def smart_decorator(f, create_decorator):
         return create_decorator(f.__func__.__call__, True)
 
 
-def inline_args(f):
-    def create_decorator(_f, with_self):
-        if with_self:
-            def f(self, args):
-                return _f(self, *args)
-        else:
-            def f(args):
-                return _f(*args)
-
-    return smart_decorator(f, create_decorator)
 
 
 try:

tests/test_parser.py

@@ -21,8 +21,7 @@ from lark.lark import Lark
 from lark.common import GrammarError, ParseError, UnexpectedToken
 from lark.lexer import LexError, UnexpectedInput
 from lark.tree import Tree
-from lark.transformers import ChildrenTransformer as Transformer
-# from lark.tree import Transformer
+from lark.transformers import Transformer, children_args
 
 __path__ = os.path.dirname(__file__)
 def _read(n, *args):
@@ -94,6 +93,7 @@ class TestParsers(unittest.TestCase):
         self.assertEqual( r.children[0].data, "c" )
 
     def test_embedded_transformer(self):
+        @children_args
         class T(Transformer):
             def a(self, children):
                 return "<a>"