From c3bce19dc274b2031c0ee88af6b6bbd065c707da Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Fri, 13 Apr 2018 13:41:30 +0300 Subject: [PATCH] More steps towards a good solution --- lark/load_grammar.py | 27 ++++++++------- lark/parsers/earley.py | 4 +-- lark/transformers.py | 74 ++++++++++++++++++------------------------ tests/test_parser.py | 2 +- 4 files changed, 47 insertions(+), 60 deletions(-) diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 190bda6..e29c47e 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -15,7 +15,7 @@ from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr from .grammar import RuleOptions, Rule from .tree import Tree, SlottedTree as ST -from .transformers import Transformer, Transformer_Children, Transformer_ChildrenInline, Visitor +from .transformers import Transformer, ChildrenTransformer, inline_args, Visitor __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -131,7 +131,8 @@ RULES = { } -class EBNF_to_BNF(Transformer_ChildrenInline): +@inline_args +class EBNF_to_BNF(ChildrenTransformer): def __init__(self): self.new_rules = [] self.rules_by_expr = {} @@ -224,7 +225,7 @@ class SimplifyRule_Visitor(Visitor): tree.children = list(set(tree.children)) -class RuleTreeToText(Transformer_Children): +class RuleTreeToText(ChildrenTransformer): def expansions(self, x): return x def expansion(self, symbols): @@ -235,17 +236,13 @@ class RuleTreeToText(Transformer_Children): return expansion, alias.value -class CanonizeTree(Transformer_ChildrenInline): +@inline_args +class CanonizeTree(ChildrenTransformer): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) - def tokenmods(self, *args): - if len(args) == 1: - return list(args) - tokenmods, value = args - return tokenmods + [value] - -class ExtractAnonTokens(Transformer_ChildrenInline): +@inline_args +class ExtractAnonTokens(ChildrenTransformer): "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them" def __init__(self, tokens): @@ -349,7 +346,8 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -class PrepareLiterals(Transformer_ChildrenInline): +@inline_args +class PrepareLiterals(ChildrenTransformer): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -361,13 +359,14 @@ class PrepareLiterals(Transformer_ChildrenInline): regexp = '[%s-%s]' % (start, end) return ST('pattern', [PatternRE(regexp)]) -class SplitLiterals(Transformer_ChildrenInline): +@inline_args +class SplitLiterals(ChildrenTransformer): def pattern(self, p): if isinstance(p, PatternStr) and len(p.value)>1: return ST('expansion', [ST('pattern', [PatternStr(ch, flags=p.flags)]) for ch in p.value]) return ST('pattern', [p]) -class TokenTreeToPattern(Transformer_Children): +class TokenTreeToPattern(ChildrenTransformer): def pattern(self, ps): p ,= ps return p diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index ee9f871..15d42a8 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -15,7 +15,7 @@ from ..common import ParseError, UnexpectedToken, is_terminal from ..tree import Tree -from ..transformers import InPlaceTransformer +from ..transformers import Transformer_InPlace from .grammar_analysis import GrammarAnalyzer @@ -230,7 +230,7 @@ class Parser: return ApplyCallbacks(self.postprocess).transform(tree) -class ApplyCallbacks(InPlaceTransformer): +class ApplyCallbacks(Transformer_InPlace): def __init__(self, postprocess): self.postprocess = postprocess diff --git a/lark/transformers.py b/lark/transformers.py index 245b733..b9f63ce 100644 --- a/lark/transformers.py +++ b/lark/transformers.py @@ -1,5 +1,7 @@ +import inspect from functools import wraps +from . import utils from .tree import Tree class Discard(Exception): @@ -31,7 +33,7 @@ class Transformer(Base): def __mul__(self, other): return TransformerChain(self, other) -class Transformer_Children(Transformer): +class ChildrenTransformer(Transformer): def _call_userfunc(self, tree): # Assumes tree is already transformed try: @@ -41,7 +43,7 @@ class Transformer_Children(Transformer): else: return f(tree.children) -class Transformer_ChildrenInline(Transformer): +class ChildrenInlineTransformer(Transformer): def _call_userfunc(self, tree): # Assumes tree is already transformed try: @@ -64,58 +66,44 @@ class TransformerChain(object): def __mul__(self, other): return TransformerChain(*self.transformers + (other,)) -class Visitor(Base): - # def visit(self, tree): - # for child in tree.children: - # if isinstance(child, Tree): - # self.visit(child) - # f = getattr(self, tree.data, self.__default__) - # f(tree) - # return tree +class Transformer_InPlace(Transformer): + def _transform(self, tree): + return self._call_userfunc(tree) + + def transform(self, tree): + for subtree in tree.iter_subtrees(): + subtree.children = list(self._transform_children(subtree.children)) + + return self._transform(tree) +class Visitor(Base): def visit(self, tree): for subtree in tree.iter_subtrees(): self._call_userfunc(subtree) return tree - -class InPlaceTransformer(Transformer): - # def _transform(self, tree): - # children = [] - # for c in tree.children: - # try: - # children.append(self._transform(c) if isinstance(c, Tree) else c) - # except Discard: - # pass - - # tree.children = children - # return self._call_userfunc(tree) - +class Transformer_InPlaceRecursive(Transformer): def _transform(self, tree): + tree.children = list(self._transform_children(tree.children)) return self._call_userfunc(tree) - def transform(self, tree): - for subtree in tree.iter_subtrees(): - subtree.children = list(self._transform_children(subtree.children)) - - return self._transform(tree) +class Visitor_Recursive(Base): + def visit(self, tree): + for child in tree.children: + if isinstance(child, Tree): + self.visit(child) + f = getattr(self, tree.data, self.__default__) + f(tree) + return tree -#### XXX PSEUDOCODE TODO -# def items(obj): -# if isinstance(obj, Transformer): -# def new_get_userfunc(self, name): -# uf = self._get_userfunc(name) -# def _f(tree): -# return uf(tree.children) -# return _f -# obj._get_userfunc = new_get_userfunc -# else: -# assert callable(obj) -# # apply decorator -# def _f(tree): -# return obj(tree.children) -# return _f +def inline_args(obj): + if inspect.isclass(obj) and issubclass(obj, ChildrenTransformer): + class _NewTransformer(ChildrenInlineTransformer, obj): + pass + return _NewTransformer + else: + return utils.inline_args(obj) diff --git a/tests/test_parser.py b/tests/test_parser.py index 6ffe1ad..7620241 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,7 +21,7 @@ from lark.lark import Lark from lark.common import GrammarError, ParseError, UnexpectedToken from lark.lexer import LexError, UnexpectedInput from lark.tree import Tree -from lark.transformers import Transformer_Children as Transformer +from lark.transformers import ChildrenTransformer as Transformer # from lark.tree import Transformer __path__ = os.path.dirname(__file__)