Initial: Added transformers.py, and Meta to tree

lark/lexer.py

@@ -34,6 +34,8 @@ class Token(Str):
         self.value = value
         self.line = line
         self.column = column
+        self.end_line = None
+        self.end_column = None
         return self
 
     @classmethod
@@ -112,6 +114,7 @@ class _Lex:
                     if t:
                         t.end_line = line_ctr.line
                         t.end_column = line_ctr.column
+
                     break
             else:
                 if line_ctr.char_pos < len(stream):

lark/load_grammar.py

@@ -14,7 +14,8 @@ from .parsers.lalr_parser import UnexpectedToken
 from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef
 from .grammar import RuleOptions, Rule
 
-from .tree import Tree, Transformer, InlineTransformer, Visitor, SlottedTree as ST
+from .tree import Tree, Visitor, SlottedTree as ST
+from .transformers import Transformer_Children, Transformer_ChildrenInline
 
 __path__ = os.path.dirname(__file__)
 IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
@@ -130,7 +131,7 @@ RULES = {
 }
 
 
-class EBNF_to_BNF(InlineTransformer):
+class EBNF_to_BNF(Transformer_ChildrenInline):
     def __init__(self):
         self.new_rules = []
         self.rules_by_expr = {}
@@ -226,7 +227,7 @@ class SimplifyRule_Visitor(Visitor):
         tree.children = list(set(tree.children))
 
 
-class RuleTreeToText(Transformer):
+class RuleTreeToText(Transformer_Children):
     def expansions(self, x):
         return x
     def expansion(self, symbols):
@@ -237,7 +238,7 @@ class RuleTreeToText(Transformer):
         return expansion, alias.value
 
 
-class CanonizeTree(InlineTransformer):
+class CanonizeTree(Transformer_ChildrenInline):
     def maybe(self, expr):
         return ST('expr', [expr, Token('OP', '?', -1)])
 
@@ -247,7 +248,7 @@ class CanonizeTree(InlineTransformer):
         tokenmods, value = args
         return tokenmods + [value]
 
-class ExtractAnonTokens(InlineTransformer):
+class ExtractAnonTokens(Transformer_ChildrenInline):
     "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them"
 
     def __init__(self, tokens):
@@ -351,7 +352,7 @@ def _literal_to_pattern(literal):
              'REGEXP': PatternRE }[literal.type](s, flags)
 
 
-class PrepareLiterals(InlineTransformer):
+class PrepareLiterals(Transformer_ChildrenInline):
     def literal(self, literal):
         return ST('pattern', [_literal_to_pattern(literal)])
 
@@ -363,13 +364,13 @@ class PrepareLiterals(InlineTransformer):
         regexp = '[%s-%s]' % (start, end)
         return ST('pattern', [PatternRE(regexp)])
 
-class SplitLiterals(InlineTransformer):
+class SplitLiterals(Transformer_ChildrenInline):
     def pattern(self, p):
         if isinstance(p, PatternStr) and len(p.value)>1:
             return ST('expansion', [ST('pattern', [PatternStr(ch, flags=p.flags)]) for ch in p.value])
         return ST('pattern', [p])
 
-class TokenTreeToPattern(Transformer):
+class TokenTreeToPattern(Transformer_Children):
     def pattern(self, ps):
         p ,= ps
         return p

lark/parse_tree_builder.py

@@ -2,6 +2,7 @@ from .common import is_terminal, GrammarError
 from .utils import suppress
 from .lexer import Token
 from .grammar import Rule
+from .tree import Tree
 
 ###{standalone
 from functools import partial
@@ -38,15 +39,23 @@ class PropagatePositions:
 
         if children:
             for a in children:
-                with suppress(AttributeError):
-                    res.line = a.line
-                    res.column = a.column
+                if isinstance(a, Tree):
+                    res.meta.line = a.meta.line
+                    res.meta.column = a.meta.column
+                elif isinstance(a, Token):
+                    res.meta.line = a.line
+                    res.meta.column = a.column
                 break
 
             for a in reversed(children):
-                with suppress(AttributeError):
-                    res.end_line = a.end_line
-                    res.end_column = a.end_column
+                # with suppress(AttributeError):
+                if isinstance(a, Tree):
+                    res.meta.end_line = a.meta.end_line
+                    res.meta.end_column = a.meta.end_column
+                elif isinstance(a, Token):
+                    res.meta.end_line = a.end_line
+                    res.meta.end_column = a.end_column
+
                 break
 
         return res

lark/transformers.py

@@ -0,0 +1,93 @@
+from functools import wraps
+
+from .tree import Tree
+
+class Discard(Exception):
+    pass
+
+
+class Transformer:
+    def _get_userfunc(self, name):
+        return getattr(self, name)
+
+    def _call_userfunc(self, tree):
+        # Assumes tree is already transformed
+        try:
+            f = self._get_userfunc(tree.data)
+        except AttributeError:
+            return self.__default__(tree)
+        else:
+            return f(tree)
+
+    def _transform(self, tree):
+        children = []
+        for c in tree.children:
+            try:
+                children.append(self._transform(c) if isinstance(c, Tree) else c)
+            except Discard:
+                pass
+
+        tree = Tree(tree.data, children)
+        return self._call_userfunc(tree)
+
+    def __default__(self, tree):
+        return tree
+
+    def transform(self, tree):
+        return self._transform(tree)
+
+    def __mul__(self, other):
+        return TransformerChain(self, other)
+
+class Transformer_Children(Transformer):
+    def _call_userfunc(self, tree):
+        # Assumes tree is already transformed
+        try:
+            f = self._get_userfunc(tree.data)
+        except AttributeError:
+            return self.__default__(tree)
+        else:
+            return f(tree.children)
+
+class Transformer_ChildrenInline(Transformer):
+    def _call_userfunc(self, tree):
+        # Assumes tree is already transformed
+        try:
+            f = self._get_userfunc(tree.data)
+        except AttributeError:
+            return self.__default__(tree)
+        else:
+            return f(*tree.children)
+
+
+class TransformerChain(object):
+    def __init__(self, *transformers):
+        self.transformers = transformers
+
+    def transform(self, tree):
+        for t in self.transformers:
+            tree = t.transform(tree)
+        return tree
+
+    def __mul__(self, other):
+        return TransformerChain(*self.transformers + (other,))
+
+
+
+#### XXX PSEUDOCODE TODO
+# def items(obj):
+#     if isinstance(obj, Transformer):
+#         def new_get_userfunc(self, name):
+#             uf = self._get_userfunc(name)
+#             def _f(tree):
+#                 return uf(tree.children)
+#             return _f
+#         obj._get_userfunc = new_get_userfunc
+#     else:
+#         assert callable(obj)
+#         # apply decorator
+#         def _f(tree):
+#             return obj(tree.children)
+#         return _f
+
+

lark/tree.py

@@ -7,11 +7,23 @@ from copy import deepcopy
 
 from .utils import inline_args
 
+class Meta:
+    pass
+
 ###{standalone
 class Tree(object):
+    __slots__ = ('data', 'children', '_meta', 'rule')
+
     def __init__(self, data, children):
         self.data = data
         self.children = children
+        self._meta = None
+
+    @property
+    def meta(self):
+        if self._meta is None:
+            self._meta = Meta()
+        return self._meta
 
     def __repr__(self):
         return 'Tree(%s, %s)' % (self.data, self.children)

tests/test_parser.py

@@ -60,7 +60,7 @@ class TestParsers(unittest.TestCase):
                  """, propagate_positions=True)
 
         r = g.parse('a')
-        self.assertEqual( r.children[0].line, 1 )
+        self.assertEqual( r.children[0].meta.line, 1 )
 
     def test_expand1(self):