More steps towards a good solution

lark/load_grammar.py

@@ -15,7 +15,7 @@ from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr
 from .grammar import RuleOptions, Rule
 
 from .tree import Tree, SlottedTree as ST
-from .transformers import Transformer, Transformer_Children, Transformer_ChildrenInline, Visitor
+from .transformers import Transformer, ChildrenTransformer, inline_args, Visitor
 
 __path__ = os.path.dirname(__file__)
 IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
@@ -131,7 +131,8 @@ RULES = {
 }
 
 
-class EBNF_to_BNF(Transformer_ChildrenInline):
+@inline_args
+class EBNF_to_BNF(ChildrenTransformer):
     def __init__(self):
         self.new_rules = []
         self.rules_by_expr = {}
@@ -224,7 +225,7 @@ class SimplifyRule_Visitor(Visitor):
         tree.children = list(set(tree.children))
 
 
-class RuleTreeToText(Transformer_Children):
+class RuleTreeToText(ChildrenTransformer):
     def expansions(self, x):
         return x
     def expansion(self, symbols):
@@ -235,17 +236,13 @@ class RuleTreeToText(Transformer_Children):
         return expansion, alias.value
 
 
-class CanonizeTree(Transformer_ChildrenInline):
+@inline_args
+class CanonizeTree(ChildrenTransformer):
     def maybe(self, expr):
         return ST('expr', [expr, Token('OP', '?', -1)])
 
-    def tokenmods(self, *args):
-        if len(args) == 1:
-            return list(args)
-        tokenmods, value = args
-        return tokenmods + [value]
-
-class ExtractAnonTokens(Transformer_ChildrenInline):
+@inline_args
+class ExtractAnonTokens(ChildrenTransformer):
     "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them"
 
     def __init__(self, tokens):
@@ -349,7 +346,8 @@ def _literal_to_pattern(literal):
              'REGEXP': PatternRE }[literal.type](s, flags)
 
 
-class PrepareLiterals(Transformer_ChildrenInline):
+@inline_args
+class PrepareLiterals(ChildrenTransformer):
     def literal(self, literal):
         return ST('pattern', [_literal_to_pattern(literal)])
 
@@ -361,13 +359,14 @@ class PrepareLiterals(Transformer_ChildrenInline):
         regexp = '[%s-%s]' % (start, end)
         return ST('pattern', [PatternRE(regexp)])
 
-class SplitLiterals(Transformer_ChildrenInline):
+@inline_args
+class SplitLiterals(ChildrenTransformer):
     def pattern(self, p):
         if isinstance(p, PatternStr) and len(p.value)>1:
             return ST('expansion', [ST('pattern', [PatternStr(ch, flags=p.flags)]) for ch in p.value])
         return ST('pattern', [p])
 
-class TokenTreeToPattern(Transformer_Children):
+class TokenTreeToPattern(ChildrenTransformer):
     def pattern(self, ps):
         p ,= ps
         return p

lark/parsers/earley.py

@@ -15,7 +15,7 @@
 
 from ..common import ParseError, UnexpectedToken, is_terminal
 from ..tree import Tree
-from ..transformers import InPlaceTransformer
+from ..transformers import Transformer_InPlace
 from .grammar_analysis import GrammarAnalyzer
 
 
@@ -230,7 +230,7 @@ class Parser:
         return ApplyCallbacks(self.postprocess).transform(tree)
 
 
-class ApplyCallbacks(InPlaceTransformer):
+class ApplyCallbacks(Transformer_InPlace):
     def __init__(self, postprocess):
         self.postprocess = postprocess
 

lark/transformers.py

@@ -1,5 +1,7 @@
+import inspect
 from functools import wraps
 
+from . import utils
 from .tree import Tree
 
 class Discard(Exception):
@@ -31,7 +33,7 @@ class Transformer(Base):
     def __mul__(self, other):
         return TransformerChain(self, other)
 
-class Transformer_Children(Transformer):
+class ChildrenTransformer(Transformer):
     def _call_userfunc(self, tree):
         # Assumes tree is already transformed
         try:
@@ -41,7 +43,7 @@ class Transformer_Children(Transformer):
         else:
             return f(tree.children)
 
-class Transformer_ChildrenInline(Transformer):
+class ChildrenInlineTransformer(Transformer):
     def _call_userfunc(self, tree):
         # Assumes tree is already transformed
         try:
@@ -64,58 +66,44 @@ class TransformerChain(object):
     def __mul__(self, other):
         return TransformerChain(*self.transformers + (other,))
 
-class Visitor(Base):
-    # def visit(self, tree):
-    #     for child in tree.children:
-    #         if isinstance(child, Tree):
-    #             self.visit(child)
 
-    #     f = getattr(self, tree.data, self.__default__)
-    #     f(tree)
-    #     return tree
+class Transformer_InPlace(Transformer):
+    def _transform(self, tree):
+        return self._call_userfunc(tree)
+
+    def transform(self, tree):
+        for subtree in tree.iter_subtrees():
+            subtree.children = list(self._transform_children(subtree.children))
+
+        return self._transform(tree)
 
+class Visitor(Base):
     def visit(self, tree):
         for subtree in tree.iter_subtrees():
             self._call_userfunc(subtree)
         return tree
 
-
-class InPlaceTransformer(Transformer):
-    # def _transform(self, tree):
-    #     children = []
-    #     for c in tree.children:
-    #         try:
-    #             children.append(self._transform(c) if isinstance(c, Tree) else c)
-    #         except Discard:
-    #             pass
-
-    #     tree.children = children
-    #     return self._call_userfunc(tree)
-
+class Transformer_InPlaceRecursive(Transformer):
     def _transform(self, tree):
+        tree.children = list(self._transform_children(tree.children))
         return self._call_userfunc(tree)
 
-    def transform(self, tree):
-        for subtree in tree.iter_subtrees():
-            subtree.children = list(self._transform_children(subtree.children))
-
-        return self._transform(tree)
+class Visitor_Recursive(Base):
+    def visit(self, tree):
+        for child in tree.children:
+            if isinstance(child, Tree):
+                self.visit(child)
 
+        f = getattr(self, tree.data, self.__default__)
+        f(tree)
+        return tree
 
-#### XXX PSEUDOCODE TODO
-# def items(obj):
-#     if isinstance(obj, Transformer):
-#         def new_get_userfunc(self, name):
-#             uf = self._get_userfunc(name)
-#             def _f(tree):
-#                 return uf(tree.children)
-#             return _f
-#         obj._get_userfunc = new_get_userfunc
-#     else:
-#         assert callable(obj)
-#         # apply decorator
-#         def _f(tree):
-#             return obj(tree.children)
-#         return _f
 
+def inline_args(obj):
+    if inspect.isclass(obj) and issubclass(obj, ChildrenTransformer):
+        class _NewTransformer(ChildrenInlineTransformer, obj):
+            pass
+        return _NewTransformer
+    else:
+        return utils.inline_args(obj)
 

tests/test_parser.py

@@ -21,7 +21,7 @@ from lark.lark import Lark
 from lark.common import GrammarError, ParseError, UnexpectedToken
 from lark.lexer import LexError, UnexpectedInput
 from lark.tree import Tree
-from lark.transformers import Transformer_Children as Transformer
+from lark.transformers import ChildrenTransformer as Transformer
 # from lark.tree import Transformer
 
 __path__ = os.path.dirname(__file__)