diff --git a/lark/lexer.py b/lark/lexer.py index 0a46ee1..1c1b70a 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -34,6 +34,8 @@ class Token(Str): self.value = value self.line = line self.column = column + self.end_line = None + self.end_column = None return self @classmethod @@ -112,6 +114,7 @@ class _Lex: if t: t.end_line = line_ctr.line t.end_column = line_ctr.column + break else: if line_ctr.char_pos < len(stream): diff --git a/lark/load_grammar.py b/lark/load_grammar.py index a6b2d82..5813708 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -14,7 +14,8 @@ from .parsers.lalr_parser import UnexpectedToken from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef from .grammar import RuleOptions, Rule -from .tree import Tree, Transformer, InlineTransformer, Visitor, SlottedTree as ST +from .tree import Tree, Visitor, SlottedTree as ST +from .transformers import Transformer_Children, Transformer_ChildrenInline __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -130,7 +131,7 @@ RULES = { } -class EBNF_to_BNF(InlineTransformer): +class EBNF_to_BNF(Transformer_ChildrenInline): def __init__(self): self.new_rules = [] self.rules_by_expr = {} @@ -226,7 +227,7 @@ class SimplifyRule_Visitor(Visitor): tree.children = list(set(tree.children)) -class RuleTreeToText(Transformer): +class RuleTreeToText(Transformer_Children): def expansions(self, x): return x def expansion(self, symbols): @@ -237,7 +238,7 @@ class RuleTreeToText(Transformer): return expansion, alias.value -class CanonizeTree(InlineTransformer): +class CanonizeTree(Transformer_ChildrenInline): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) @@ -247,7 +248,7 @@ class CanonizeTree(InlineTransformer): tokenmods, value = args return tokenmods + [value] -class ExtractAnonTokens(InlineTransformer): +class ExtractAnonTokens(Transformer_ChildrenInline): "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them" def __init__(self, tokens): @@ -351,7 +352,7 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -class PrepareLiterals(InlineTransformer): +class PrepareLiterals(Transformer_ChildrenInline): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -363,13 +364,13 @@ class PrepareLiterals(InlineTransformer): regexp = '[%s-%s]' % (start, end) return ST('pattern', [PatternRE(regexp)]) -class SplitLiterals(InlineTransformer): +class SplitLiterals(Transformer_ChildrenInline): def pattern(self, p): if isinstance(p, PatternStr) and len(p.value)>1: return ST('expansion', [ST('pattern', [PatternStr(ch, flags=p.flags)]) for ch in p.value]) return ST('pattern', [p]) -class TokenTreeToPattern(Transformer): +class TokenTreeToPattern(Transformer_Children): def pattern(self, ps): p ,= ps return p diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 7c74178..1acfe2f 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -2,6 +2,7 @@ from .common import is_terminal, GrammarError from .utils import suppress from .lexer import Token from .grammar import Rule +from .tree import Tree ###{standalone from functools import partial @@ -38,15 +39,23 @@ class PropagatePositions: if children: for a in children: - with suppress(AttributeError): - res.line = a.line - res.column = a.column + if isinstance(a, Tree): + res.meta.line = a.meta.line + res.meta.column = a.meta.column + elif isinstance(a, Token): + res.meta.line = a.line + res.meta.column = a.column break for a in reversed(children): - with suppress(AttributeError): - res.end_line = a.end_line - res.end_column = a.end_column + # with suppress(AttributeError): + if isinstance(a, Tree): + res.meta.end_line = a.meta.end_line + res.meta.end_column = a.meta.end_column + elif isinstance(a, Token): + res.meta.end_line = a.end_line + res.meta.end_column = a.end_column + break return res diff --git a/lark/transformers.py b/lark/transformers.py new file mode 100644 index 0000000..b1938f0 --- /dev/null +++ b/lark/transformers.py @@ -0,0 +1,93 @@ +from functools import wraps + +from .tree import Tree + +class Discard(Exception): + pass + + +class Transformer: + def _get_userfunc(self, name): + return getattr(self, name) + + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = self._get_userfunc(tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(tree) + + def _transform(self, tree): + children = [] + for c in tree.children: + try: + children.append(self._transform(c) if isinstance(c, Tree) else c) + except Discard: + pass + + tree = Tree(tree.data, children) + return self._call_userfunc(tree) + + def __default__(self, tree): + return tree + + def transform(self, tree): + return self._transform(tree) + + def __mul__(self, other): + return TransformerChain(self, other) + +class Transformer_Children(Transformer): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = self._get_userfunc(tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(tree.children) + +class Transformer_ChildrenInline(Transformer): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = self._get_userfunc(tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(*tree.children) + + +class TransformerChain(object): + def __init__(self, *transformers): + self.transformers = transformers + + def transform(self, tree): + for t in self.transformers: + tree = t.transform(tree) + return tree + + def __mul__(self, other): + return TransformerChain(*self.transformers + (other,)) + + + +#### XXX PSEUDOCODE TODO +# def items(obj): +# if isinstance(obj, Transformer): +# def new_get_userfunc(self, name): +# uf = self._get_userfunc(name) +# def _f(tree): +# return uf(tree.children) +# return _f +# obj._get_userfunc = new_get_userfunc +# else: +# assert callable(obj) +# # apply decorator +# def _f(tree): +# return obj(tree.children) +# return _f + + diff --git a/lark/tree.py b/lark/tree.py index d496d75..04cfc4e 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -7,11 +7,23 @@ from copy import deepcopy from .utils import inline_args +class Meta: + pass + ###{standalone class Tree(object): + __slots__ = ('data', 'children', '_meta', 'rule') + def __init__(self, data, children): self.data = data self.children = children + self._meta = None + + @property + def meta(self): + if self._meta is None: + self._meta = Meta() + return self._meta def __repr__(self): return 'Tree(%s, %s)' % (self.data, self.children) diff --git a/tests/test_parser.py b/tests/test_parser.py index d4d63ca..4aaea93 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -60,7 +60,7 @@ class TestParsers(unittest.TestCase): """, propagate_positions=True) r = g.parse('a') - self.assertEqual( r.children[0].line, 1 ) + self.assertEqual( r.children[0].meta.line, 1 ) def test_expand1(self):