diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 5813708..190bda6 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -14,8 +14,8 @@ from .parsers.lalr_parser import UnexpectedToken from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef from .grammar import RuleOptions, Rule -from .tree import Tree, Visitor, SlottedTree as ST -from .transformers import Transformer_Children, Transformer_ChildrenInline +from .tree import Tree, SlottedTree as ST +from .transformers import Transformer, Transformer_Children, Transformer_ChildrenInline, Visitor __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -200,17 +200,14 @@ class SimplifyRule_Visitor(Visitor): # --> # expansions( expansion(b, c, e), expansion(b, d, e) ) - while True: - self._flatten(tree) - - for i, child in enumerate(tree.children): - if isinstance(child, Tree) and child.data == 'expansions': - tree.data = 'expansions' - tree.children = [self.visit(ST('expansion', [option if i==j else other - for j, other in enumerate(tree.children)])) - for option in set(child.children)] - break - else: + self._flatten(tree) + + for i, child in enumerate(tree.children): + if isinstance(child, Tree) and child.data == 'expansions': + tree.data = 'expansions' + tree.children = [self.visit(ST('expansion', [option if i==j else other + for j, other in enumerate(tree.children)])) + for option in set(child.children)] break def alias(self, tree): @@ -234,7 +231,7 @@ class RuleTreeToText(Transformer_Children): return [sym.value for sym in symbols], None def alias(self, x): (expansion, _alias), alias = x - assert _alias is None, (alias, expansion, '-', _alias) + assert _alias is None, (alias, expansion, '-', _alias) # Double alias not allowed return expansion, alias.value diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index d119e41..ee9f871 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,8 @@ # Email : erezshin@gmail.com from ..common import ParseError, UnexpectedToken, is_terminal -from ..tree import Tree, Transformer_NoRecurse +from ..tree import Tree +from ..transformers import InPlaceTransformer from .grammar_analysis import GrammarAnalyzer @@ -229,7 +230,7 @@ class Parser: return ApplyCallbacks(self.postprocess).transform(tree) -class ApplyCallbacks(Transformer_NoRecurse): +class ApplyCallbacks(InPlaceTransformer): def __init__(self, postprocess): self.postprocess = postprocess diff --git a/lark/parsers/resolve_ambig.py b/lark/parsers/resolve_ambig.py index 456c6a9..7c482ae 100644 --- a/lark/parsers/resolve_ambig.py +++ b/lark/parsers/resolve_ambig.py @@ -1,7 +1,7 @@ from ..utils import compare from functools import cmp_to_key -from ..tree import Tree, Visitor_NoRecurse +from ..tree import Tree # Standard ambiguity resolver (uses comparison) diff --git a/lark/transformers.py b/lark/transformers.py index b1938f0..033d2f4 100644 --- a/lark/transformers.py +++ b/lark/transformers.py @@ -6,33 +6,25 @@ class Discard(Exception): pass -class Transformer: - def _get_userfunc(self, name): - return getattr(self, name) - +class Base: def _call_userfunc(self, tree): - # Assumes tree is already transformed - try: - f = self._get_userfunc(tree.data) - except AttributeError: - return self.__default__(tree) - else: - return f(tree) + return getattr(self, tree.data, self.__default__)(tree) - def _transform(self, tree): - children = [] - for c in tree.children: + def __default__(self, tree): + return tree + +class Transformer(Base): + def _transform_children(self, children): + for c in children: try: - children.append(self._transform(c) if isinstance(c, Tree) else c) + yield self._transform(c) if isinstance(c, Tree) else c except Discard: pass - tree = Tree(tree.data, children) + def _transform(self, tree): + tree = Tree(tree.data, list(self._transform_children(tree.children))) return self._call_userfunc(tree) - def __default__(self, tree): - return tree - def transform(self, tree): return self._transform(tree) @@ -43,7 +35,7 @@ class Transformer_Children(Transformer): def _call_userfunc(self, tree): # Assumes tree is already transformed try: - f = self._get_userfunc(tree.data) + f = getattr(self, tree.data) except AttributeError: return self.__default__(tree) else: @@ -53,7 +45,7 @@ class Transformer_ChildrenInline(Transformer): def _call_userfunc(self, tree): # Assumes tree is already transformed try: - f = self._get_userfunc(tree.data) + f = getattr(self, tree.data) except AttributeError: return self.__default__(tree) else: @@ -72,6 +64,45 @@ class TransformerChain(object): def __mul__(self, other): return TransformerChain(*self.transformers + (other,)) +class Visitor(Base): + # def visit(self, tree): + # for child in tree.children: + # if isinstance(child, Tree): + # self.visit(child) + + # f = getattr(self, tree.data, self.__default__) + # f(tree) + # return tree + + def visit(self, tree): + for subtree in tree.iter_subtrees(): + self._call_userfunc(subtree) + return tree + + def __default__(self, tree): + pass + + +class InPlaceTransformer(Transformer): + # def _transform(self, tree): + # children = [] + # for c in tree.children: + # try: + # children.append(self._transform(c) if isinstance(c, Tree) else c) + # except Discard: + # pass + + # tree.children = children + # return self._call_userfunc(tree) + + def _transform(self, tree): + return self._call_userfunc(tree) + + def transform(self, tree): + for subtree in tree.iter_subtrees(): + subtree.children = list(self._transform_children(subtree.children)) + + return self._transform(tree) #### XXX PSEUDOCODE TODO diff --git a/lark/tree.py b/lark/tree.py index 04cfc4e..cf88293 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -12,8 +12,6 @@ class Meta: ###{standalone class Tree(object): - __slots__ = ('data', 'children', '_meta', 'rule') - def __init__(self, data, children): self.data = data self.children = children @@ -141,77 +139,12 @@ class Transformer(object): return TransformerChain(self, other) -class Discard(Exception): - pass - -class TransformerChain(object): - def __init__(self, *transformers): - self.transformers = transformers - - def transform(self, tree): - for t in self.transformers: - tree = t.transform(tree) - return tree - - def __mul__(self, other): - return TransformerChain(*self.transformers + (other,)) - - - class InlineTransformer(Transformer): def _get_func(self, name): # use super()._get_func return inline_args(getattr(self, name)).__get__(self) -class Visitor(object): - def visit(self, tree): - for child in tree.children: - if isinstance(child, Tree): - self.visit(child) - - f = getattr(self, tree.data, self.__default__) - f(tree) - return tree - - def __default__(self, tree): - pass - - -class Visitor_NoRecurse(Visitor): - def visit(self, tree): - subtrees = list(tree.iter_subtrees()) - - for subtree in (subtrees): - getattr(self, subtree.data, self.__default__)(subtree) - return tree - - -class Transformer_NoRecurse(Transformer): - def transform(self, tree): - subtrees = list(tree.iter_subtrees()) - - def _t(t): - # Assumes t is already transformed - try: - f = self._get_func(t.data) - except AttributeError: - return self.__default__(t) - else: - return f(t) - - for subtree in subtrees: - children = [] - for c in subtree.children: - try: - children.append(_t(c) if isinstance(c, Tree) else c) - except Discard: - pass - subtree.children = children - - return _t(tree) - def __default__(self, t): - return t ###}