From f960c1b8ac0fb77d821ac2c1462910992fdc74e4 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Fri, 13 Apr 2018 00:40:28 +0300 Subject: [PATCH 01/12] Initial: Added transformers.py, and Meta to tree --- lark/lexer.py | 3 ++ lark/load_grammar.py | 17 +++---- lark/parse_tree_builder.py | 21 ++++++--- lark/transformers.py | 93 ++++++++++++++++++++++++++++++++++++++ lark/tree.py | 12 +++++ tests/test_parser.py | 2 +- 6 files changed, 133 insertions(+), 15 deletions(-) create mode 100644 lark/transformers.py diff --git a/lark/lexer.py b/lark/lexer.py index 0a46ee1..1c1b70a 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -34,6 +34,8 @@ class Token(Str): self.value = value self.line = line self.column = column + self.end_line = None + self.end_column = None return self @classmethod @@ -112,6 +114,7 @@ class _Lex: if t: t.end_line = line_ctr.line t.end_column = line_ctr.column + break else: if line_ctr.char_pos < len(stream): diff --git a/lark/load_grammar.py b/lark/load_grammar.py index a6b2d82..5813708 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -14,7 +14,8 @@ from .parsers.lalr_parser import UnexpectedToken from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef from .grammar import RuleOptions, Rule -from .tree import Tree, Transformer, InlineTransformer, Visitor, SlottedTree as ST +from .tree import Tree, Visitor, SlottedTree as ST +from .transformers import Transformer_Children, Transformer_ChildrenInline __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -130,7 +131,7 @@ RULES = { } -class EBNF_to_BNF(InlineTransformer): +class EBNF_to_BNF(Transformer_ChildrenInline): def __init__(self): self.new_rules = [] self.rules_by_expr = {} @@ -226,7 +227,7 @@ class SimplifyRule_Visitor(Visitor): tree.children = list(set(tree.children)) -class RuleTreeToText(Transformer): +class RuleTreeToText(Transformer_Children): def expansions(self, x): return x def expansion(self, symbols): @@ -237,7 +238,7 @@ class RuleTreeToText(Transformer): return expansion, alias.value -class CanonizeTree(InlineTransformer): +class CanonizeTree(Transformer_ChildrenInline): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) @@ -247,7 +248,7 @@ class CanonizeTree(InlineTransformer): tokenmods, value = args return tokenmods + [value] -class ExtractAnonTokens(InlineTransformer): +class ExtractAnonTokens(Transformer_ChildrenInline): "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them" def __init__(self, tokens): @@ -351,7 +352,7 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -class PrepareLiterals(InlineTransformer): +class PrepareLiterals(Transformer_ChildrenInline): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -363,13 +364,13 @@ class PrepareLiterals(InlineTransformer): regexp = '[%s-%s]' % (start, end) return ST('pattern', [PatternRE(regexp)]) -class SplitLiterals(InlineTransformer): +class SplitLiterals(Transformer_ChildrenInline): def pattern(self, p): if isinstance(p, PatternStr) and len(p.value)>1: return ST('expansion', [ST('pattern', [PatternStr(ch, flags=p.flags)]) for ch in p.value]) return ST('pattern', [p]) -class TokenTreeToPattern(Transformer): +class TokenTreeToPattern(Transformer_Children): def pattern(self, ps): p ,= ps return p diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 7c74178..1acfe2f 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -2,6 +2,7 @@ from .common import is_terminal, GrammarError from .utils import suppress from .lexer import Token from .grammar import Rule +from .tree import Tree ###{standalone from functools import partial @@ -38,15 +39,23 @@ class PropagatePositions: if children: for a in children: - with suppress(AttributeError): - res.line = a.line - res.column = a.column + if isinstance(a, Tree): + res.meta.line = a.meta.line + res.meta.column = a.meta.column + elif isinstance(a, Token): + res.meta.line = a.line + res.meta.column = a.column break for a in reversed(children): - with suppress(AttributeError): - res.end_line = a.end_line - res.end_column = a.end_column + # with suppress(AttributeError): + if isinstance(a, Tree): + res.meta.end_line = a.meta.end_line + res.meta.end_column = a.meta.end_column + elif isinstance(a, Token): + res.meta.end_line = a.end_line + res.meta.end_column = a.end_column + break return res diff --git a/lark/transformers.py b/lark/transformers.py new file mode 100644 index 0000000..b1938f0 --- /dev/null +++ b/lark/transformers.py @@ -0,0 +1,93 @@ +from functools import wraps + +from .tree import Tree + +class Discard(Exception): + pass + + +class Transformer: + def _get_userfunc(self, name): + return getattr(self, name) + + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = self._get_userfunc(tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(tree) + + def _transform(self, tree): + children = [] + for c in tree.children: + try: + children.append(self._transform(c) if isinstance(c, Tree) else c) + except Discard: + pass + + tree = Tree(tree.data, children) + return self._call_userfunc(tree) + + def __default__(self, tree): + return tree + + def transform(self, tree): + return self._transform(tree) + + def __mul__(self, other): + return TransformerChain(self, other) + +class Transformer_Children(Transformer): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = self._get_userfunc(tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(tree.children) + +class Transformer_ChildrenInline(Transformer): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = self._get_userfunc(tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(*tree.children) + + +class TransformerChain(object): + def __init__(self, *transformers): + self.transformers = transformers + + def transform(self, tree): + for t in self.transformers: + tree = t.transform(tree) + return tree + + def __mul__(self, other): + return TransformerChain(*self.transformers + (other,)) + + + +#### XXX PSEUDOCODE TODO +# def items(obj): +# if isinstance(obj, Transformer): +# def new_get_userfunc(self, name): +# uf = self._get_userfunc(name) +# def _f(tree): +# return uf(tree.children) +# return _f +# obj._get_userfunc = new_get_userfunc +# else: +# assert callable(obj) +# # apply decorator +# def _f(tree): +# return obj(tree.children) +# return _f + + diff --git a/lark/tree.py b/lark/tree.py index d496d75..04cfc4e 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -7,11 +7,23 @@ from copy import deepcopy from .utils import inline_args +class Meta: + pass + ###{standalone class Tree(object): + __slots__ = ('data', 'children', '_meta', 'rule') + def __init__(self, data, children): self.data = data self.children = children + self._meta = None + + @property + def meta(self): + if self._meta is None: + self._meta = Meta() + return self._meta def __repr__(self): return 'Tree(%s, %s)' % (self.data, self.children) diff --git a/tests/test_parser.py b/tests/test_parser.py index d4d63ca..4aaea93 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -60,7 +60,7 @@ class TestParsers(unittest.TestCase): """, propagate_positions=True) r = g.parse('a') - self.assertEqual( r.children[0].line, 1 ) + self.assertEqual( r.children[0].meta.line, 1 ) def test_expand1(self): From 349a607ae33de8320e54f8541ccd4255b3cb14b7 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Fri, 13 Apr 2018 11:48:41 +0300 Subject: [PATCH 02/12] Some more normalizing --- lark/load_grammar.py | 25 ++++++------ lark/parsers/earley.py | 5 ++- lark/parsers/resolve_ambig.py | 2 +- lark/transformers.py | 73 +++++++++++++++++++++++++---------- lark/tree.py | 67 -------------------------------- 5 files changed, 67 insertions(+), 105 deletions(-) diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 5813708..190bda6 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -14,8 +14,8 @@ from .parsers.lalr_parser import UnexpectedToken from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef from .grammar import RuleOptions, Rule -from .tree import Tree, Visitor, SlottedTree as ST -from .transformers import Transformer_Children, Transformer_ChildrenInline +from .tree import Tree, SlottedTree as ST +from .transformers import Transformer, Transformer_Children, Transformer_ChildrenInline, Visitor __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -200,17 +200,14 @@ class SimplifyRule_Visitor(Visitor): # --> # expansions( expansion(b, c, e), expansion(b, d, e) ) - while True: - self._flatten(tree) - - for i, child in enumerate(tree.children): - if isinstance(child, Tree) and child.data == 'expansions': - tree.data = 'expansions' - tree.children = [self.visit(ST('expansion', [option if i==j else other - for j, other in enumerate(tree.children)])) - for option in set(child.children)] - break - else: + self._flatten(tree) + + for i, child in enumerate(tree.children): + if isinstance(child, Tree) and child.data == 'expansions': + tree.data = 'expansions' + tree.children = [self.visit(ST('expansion', [option if i==j else other + for j, other in enumerate(tree.children)])) + for option in set(child.children)] break def alias(self, tree): @@ -234,7 +231,7 @@ class RuleTreeToText(Transformer_Children): return [sym.value for sym in symbols], None def alias(self, x): (expansion, _alias), alias = x - assert _alias is None, (alias, expansion, '-', _alias) + assert _alias is None, (alias, expansion, '-', _alias) # Double alias not allowed return expansion, alias.value diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index d119e41..ee9f871 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,8 @@ # Email : erezshin@gmail.com from ..common import ParseError, UnexpectedToken, is_terminal -from ..tree import Tree, Transformer_NoRecurse +from ..tree import Tree +from ..transformers import InPlaceTransformer from .grammar_analysis import GrammarAnalyzer @@ -229,7 +230,7 @@ class Parser: return ApplyCallbacks(self.postprocess).transform(tree) -class ApplyCallbacks(Transformer_NoRecurse): +class ApplyCallbacks(InPlaceTransformer): def __init__(self, postprocess): self.postprocess = postprocess diff --git a/lark/parsers/resolve_ambig.py b/lark/parsers/resolve_ambig.py index 456c6a9..7c482ae 100644 --- a/lark/parsers/resolve_ambig.py +++ b/lark/parsers/resolve_ambig.py @@ -1,7 +1,7 @@ from ..utils import compare from functools import cmp_to_key -from ..tree import Tree, Visitor_NoRecurse +from ..tree import Tree # Standard ambiguity resolver (uses comparison) diff --git a/lark/transformers.py b/lark/transformers.py index b1938f0..033d2f4 100644 --- a/lark/transformers.py +++ b/lark/transformers.py @@ -6,33 +6,25 @@ class Discard(Exception): pass -class Transformer: - def _get_userfunc(self, name): - return getattr(self, name) - +class Base: def _call_userfunc(self, tree): - # Assumes tree is already transformed - try: - f = self._get_userfunc(tree.data) - except AttributeError: - return self.__default__(tree) - else: - return f(tree) + return getattr(self, tree.data, self.__default__)(tree) - def _transform(self, tree): - children = [] - for c in tree.children: + def __default__(self, tree): + return tree + +class Transformer(Base): + def _transform_children(self, children): + for c in children: try: - children.append(self._transform(c) if isinstance(c, Tree) else c) + yield self._transform(c) if isinstance(c, Tree) else c except Discard: pass - tree = Tree(tree.data, children) + def _transform(self, tree): + tree = Tree(tree.data, list(self._transform_children(tree.children))) return self._call_userfunc(tree) - def __default__(self, tree): - return tree - def transform(self, tree): return self._transform(tree) @@ -43,7 +35,7 @@ class Transformer_Children(Transformer): def _call_userfunc(self, tree): # Assumes tree is already transformed try: - f = self._get_userfunc(tree.data) + f = getattr(self, tree.data) except AttributeError: return self.__default__(tree) else: @@ -53,7 +45,7 @@ class Transformer_ChildrenInline(Transformer): def _call_userfunc(self, tree): # Assumes tree is already transformed try: - f = self._get_userfunc(tree.data) + f = getattr(self, tree.data) except AttributeError: return self.__default__(tree) else: @@ -72,6 +64,45 @@ class TransformerChain(object): def __mul__(self, other): return TransformerChain(*self.transformers + (other,)) +class Visitor(Base): + # def visit(self, tree): + # for child in tree.children: + # if isinstance(child, Tree): + # self.visit(child) + + # f = getattr(self, tree.data, self.__default__) + # f(tree) + # return tree + + def visit(self, tree): + for subtree in tree.iter_subtrees(): + self._call_userfunc(subtree) + return tree + + def __default__(self, tree): + pass + + +class InPlaceTransformer(Transformer): + # def _transform(self, tree): + # children = [] + # for c in tree.children: + # try: + # children.append(self._transform(c) if isinstance(c, Tree) else c) + # except Discard: + # pass + + # tree.children = children + # return self._call_userfunc(tree) + + def _transform(self, tree): + return self._call_userfunc(tree) + + def transform(self, tree): + for subtree in tree.iter_subtrees(): + subtree.children = list(self._transform_children(subtree.children)) + + return self._transform(tree) #### XXX PSEUDOCODE TODO diff --git a/lark/tree.py b/lark/tree.py index 04cfc4e..cf88293 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -12,8 +12,6 @@ class Meta: ###{standalone class Tree(object): - __slots__ = ('data', 'children', '_meta', 'rule') - def __init__(self, data, children): self.data = data self.children = children @@ -141,77 +139,12 @@ class Transformer(object): return TransformerChain(self, other) -class Discard(Exception): - pass - -class TransformerChain(object): - def __init__(self, *transformers): - self.transformers = transformers - - def transform(self, tree): - for t in self.transformers: - tree = t.transform(tree) - return tree - - def __mul__(self, other): - return TransformerChain(*self.transformers + (other,)) - - - class InlineTransformer(Transformer): def _get_func(self, name): # use super()._get_func return inline_args(getattr(self, name)).__get__(self) -class Visitor(object): - def visit(self, tree): - for child in tree.children: - if isinstance(child, Tree): - self.visit(child) - - f = getattr(self, tree.data, self.__default__) - f(tree) - return tree - - def __default__(self, tree): - pass - - -class Visitor_NoRecurse(Visitor): - def visit(self, tree): - subtrees = list(tree.iter_subtrees()) - - for subtree in (subtrees): - getattr(self, subtree.data, self.__default__)(subtree) - return tree - - -class Transformer_NoRecurse(Transformer): - def transform(self, tree): - subtrees = list(tree.iter_subtrees()) - - def _t(t): - # Assumes t is already transformed - try: - f = self._get_func(t.data) - except AttributeError: - return self.__default__(t) - else: - return f(t) - - for subtree in subtrees: - children = [] - for c in subtree.children: - try: - children.append(_t(c) if isinstance(c, Tree) else c) - except Discard: - pass - subtree.children = children - - return _t(tree) - def __default__(self, t): - return t ###} From f69bceb3354c01f427535e94af15de62247c7436 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Fri, 13 Apr 2018 12:02:01 +0300 Subject: [PATCH 03/12] Snap more things into place --- lark/__init__.py | 3 ++- lark/parse_tree_builder.py | 2 +- lark/transformers.py | 3 --- lark/tree.py | 34 ---------------------------------- tests/test_parser.py | 4 +++- 5 files changed, 6 insertions(+), 40 deletions(-) diff --git a/lark/__init__.py b/lark/__init__.py index af6f7b5..a2a67b9 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,4 +1,5 @@ -from .tree import Tree, Transformer, InlineTransformer +from .tree import Tree +from .transformers import Transformer from .common import ParseError, GrammarError, UnexpectedToken from .lexer import UnexpectedInput, LexError from .lark import Lark diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 1acfe2f..ea26347 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -145,7 +145,7 @@ class ParseTreeBuilder: user_callback_name = rule.alias or rule.origin try: - f = transformer._get_func(user_callback_name) + f = getattr(transformer, user_callback_name) except AttributeError: f = partial(self.tree_class, user_callback_name) diff --git a/lark/transformers.py b/lark/transformers.py index 033d2f4..245b733 100644 --- a/lark/transformers.py +++ b/lark/transformers.py @@ -79,9 +79,6 @@ class Visitor(Base): self._call_userfunc(subtree) return tree - def __default__(self, tree): - pass - class InPlaceTransformer(Transformer): # def _transform(self, tree): diff --git a/lark/tree.py b/lark/tree.py index cf88293..c478ae6 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -113,40 +113,6 @@ class SlottedTree(Tree): __slots__ = 'data', 'children', 'rule' -###{standalone -class Transformer(object): - def _get_func(self, name): - return getattr(self, name) - - def transform(self, tree): - items = [] - for c in tree.children: - try: - items.append(self.transform(c) if isinstance(c, Tree) else c) - except Discard: - pass - try: - f = self._get_func(tree.data) - except AttributeError: - return self.__default__(tree.data, items) - else: - return f(items) - - def __default__(self, data, children): - return Tree(data, children) - - def __mul__(self, other): - return TransformerChain(self, other) - - -class InlineTransformer(Transformer): - def _get_func(self, name): # use super()._get_func - return inline_args(getattr(self, name)).__get__(self) - - - -###} - def pydot__tree_to_png(tree, filename): import pydot diff --git a/tests/test_parser.py b/tests/test_parser.py index 4aaea93..6ffe1ad 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -20,7 +20,9 @@ logging.basicConfig(level=logging.INFO) from lark.lark import Lark from lark.common import GrammarError, ParseError, UnexpectedToken from lark.lexer import LexError, UnexpectedInput -from lark.tree import Tree, Transformer +from lark.tree import Tree +from lark.transformers import Transformer_Children as Transformer +# from lark.tree import Transformer __path__ = os.path.dirname(__file__) def _read(n, *args): From c3bce19dc274b2031c0ee88af6b6bbd065c707da Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Fri, 13 Apr 2018 13:41:30 +0300 Subject: [PATCH 04/12] More steps towards a good solution --- lark/load_grammar.py | 27 ++++++++------- lark/parsers/earley.py | 4 +-- lark/transformers.py | 74 ++++++++++++++++++------------------------ tests/test_parser.py | 2 +- 4 files changed, 47 insertions(+), 60 deletions(-) diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 190bda6..e29c47e 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -15,7 +15,7 @@ from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr from .grammar import RuleOptions, Rule from .tree import Tree, SlottedTree as ST -from .transformers import Transformer, Transformer_Children, Transformer_ChildrenInline, Visitor +from .transformers import Transformer, ChildrenTransformer, inline_args, Visitor __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -131,7 +131,8 @@ RULES = { } -class EBNF_to_BNF(Transformer_ChildrenInline): +@inline_args +class EBNF_to_BNF(ChildrenTransformer): def __init__(self): self.new_rules = [] self.rules_by_expr = {} @@ -224,7 +225,7 @@ class SimplifyRule_Visitor(Visitor): tree.children = list(set(tree.children)) -class RuleTreeToText(Transformer_Children): +class RuleTreeToText(ChildrenTransformer): def expansions(self, x): return x def expansion(self, symbols): @@ -235,17 +236,13 @@ class RuleTreeToText(Transformer_Children): return expansion, alias.value -class CanonizeTree(Transformer_ChildrenInline): +@inline_args +class CanonizeTree(ChildrenTransformer): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) - def tokenmods(self, *args): - if len(args) == 1: - return list(args) - tokenmods, value = args - return tokenmods + [value] - -class ExtractAnonTokens(Transformer_ChildrenInline): +@inline_args +class ExtractAnonTokens(ChildrenTransformer): "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them" def __init__(self, tokens): @@ -349,7 +346,8 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -class PrepareLiterals(Transformer_ChildrenInline): +@inline_args +class PrepareLiterals(ChildrenTransformer): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -361,13 +359,14 @@ class PrepareLiterals(Transformer_ChildrenInline): regexp = '[%s-%s]' % (start, end) return ST('pattern', [PatternRE(regexp)]) -class SplitLiterals(Transformer_ChildrenInline): +@inline_args +class SplitLiterals(ChildrenTransformer): def pattern(self, p): if isinstance(p, PatternStr) and len(p.value)>1: return ST('expansion', [ST('pattern', [PatternStr(ch, flags=p.flags)]) for ch in p.value]) return ST('pattern', [p]) -class TokenTreeToPattern(Transformer_Children): +class TokenTreeToPattern(ChildrenTransformer): def pattern(self, ps): p ,= ps return p diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index ee9f871..15d42a8 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -15,7 +15,7 @@ from ..common import ParseError, UnexpectedToken, is_terminal from ..tree import Tree -from ..transformers import InPlaceTransformer +from ..transformers import Transformer_InPlace from .grammar_analysis import GrammarAnalyzer @@ -230,7 +230,7 @@ class Parser: return ApplyCallbacks(self.postprocess).transform(tree) -class ApplyCallbacks(InPlaceTransformer): +class ApplyCallbacks(Transformer_InPlace): def __init__(self, postprocess): self.postprocess = postprocess diff --git a/lark/transformers.py b/lark/transformers.py index 245b733..b9f63ce 100644 --- a/lark/transformers.py +++ b/lark/transformers.py @@ -1,5 +1,7 @@ +import inspect from functools import wraps +from . import utils from .tree import Tree class Discard(Exception): @@ -31,7 +33,7 @@ class Transformer(Base): def __mul__(self, other): return TransformerChain(self, other) -class Transformer_Children(Transformer): +class ChildrenTransformer(Transformer): def _call_userfunc(self, tree): # Assumes tree is already transformed try: @@ -41,7 +43,7 @@ class Transformer_Children(Transformer): else: return f(tree.children) -class Transformer_ChildrenInline(Transformer): +class ChildrenInlineTransformer(Transformer): def _call_userfunc(self, tree): # Assumes tree is already transformed try: @@ -64,58 +66,44 @@ class TransformerChain(object): def __mul__(self, other): return TransformerChain(*self.transformers + (other,)) -class Visitor(Base): - # def visit(self, tree): - # for child in tree.children: - # if isinstance(child, Tree): - # self.visit(child) - # f = getattr(self, tree.data, self.__default__) - # f(tree) - # return tree +class Transformer_InPlace(Transformer): + def _transform(self, tree): + return self._call_userfunc(tree) + + def transform(self, tree): + for subtree in tree.iter_subtrees(): + subtree.children = list(self._transform_children(subtree.children)) + + return self._transform(tree) +class Visitor(Base): def visit(self, tree): for subtree in tree.iter_subtrees(): self._call_userfunc(subtree) return tree - -class InPlaceTransformer(Transformer): - # def _transform(self, tree): - # children = [] - # for c in tree.children: - # try: - # children.append(self._transform(c) if isinstance(c, Tree) else c) - # except Discard: - # pass - - # tree.children = children - # return self._call_userfunc(tree) - +class Transformer_InPlaceRecursive(Transformer): def _transform(self, tree): + tree.children = list(self._transform_children(tree.children)) return self._call_userfunc(tree) - def transform(self, tree): - for subtree in tree.iter_subtrees(): - subtree.children = list(self._transform_children(subtree.children)) - - return self._transform(tree) +class Visitor_Recursive(Base): + def visit(self, tree): + for child in tree.children: + if isinstance(child, Tree): + self.visit(child) + f = getattr(self, tree.data, self.__default__) + f(tree) + return tree -#### XXX PSEUDOCODE TODO -# def items(obj): -# if isinstance(obj, Transformer): -# def new_get_userfunc(self, name): -# uf = self._get_userfunc(name) -# def _f(tree): -# return uf(tree.children) -# return _f -# obj._get_userfunc = new_get_userfunc -# else: -# assert callable(obj) -# # apply decorator -# def _f(tree): -# return obj(tree.children) -# return _f +def inline_args(obj): + if inspect.isclass(obj) and issubclass(obj, ChildrenTransformer): + class _NewTransformer(ChildrenInlineTransformer, obj): + pass + return _NewTransformer + else: + return utils.inline_args(obj) diff --git a/tests/test_parser.py b/tests/test_parser.py index 6ffe1ad..7620241 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,7 +21,7 @@ from lark.lark import Lark from lark.common import GrammarError, ParseError, UnexpectedToken from lark.lexer import LexError, UnexpectedInput from lark.tree import Tree -from lark.transformers import Transformer_Children as Transformer +from lark.transformers import ChildrenTransformer as Transformer # from lark.tree import Transformer __path__ = os.path.dirname(__file__) From 55e9d5679264e86c977180a6d48047174ca2ec3b Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sat, 12 May 2018 23:39:23 +0300 Subject: [PATCH 05/12] Missed those at merge --- lark/load_grammar.py | 5 +++-- lark/transformers.py | 27 +++++++++++++++++++++++++++ tests/test_trees.py | 3 ++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 5fee242..c710bf8 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -407,7 +407,7 @@ class TokenTreeToPattern(ChildrenTransformer): def value(self, v): return v[0] -class PrepareSymbols(Transformer): +class PrepareSymbols(ChildrenTransformer): def value(self, v): v ,= v if isinstance(v, Tree): @@ -532,7 +532,8 @@ def options_from_rule(name, *x): def symbols_from_strcase(expansion): return [Terminal(x, filter_out=x.startswith('_')) if is_terminal(x) else NonTerminal(x) for x in expansion] -class PrepareGrammar(InlineTransformer): +@inline_args +class PrepareGrammar(ChildrenTransformer): def terminal(self, name): return name def nonterminal(self, name): diff --git a/lark/transformers.py b/lark/transformers.py index b9f63ce..d5f13a1 100644 --- a/lark/transformers.py +++ b/lark/transformers.py @@ -78,6 +78,8 @@ class Transformer_InPlace(Transformer): return self._transform(tree) class Visitor(Base): + "Bottom-up visitor" + def visit(self, tree): for subtree in tree.iter_subtrees(): self._call_userfunc(subtree) @@ -99,6 +101,31 @@ class Visitor_Recursive(Base): return tree +from functools import wraps +def visit_children_decor(func): + @wraps(func) + def inner(cls, tree): + values = cls.visit_children(tree) + return func(cls, values) + return inner + +class Interpreter(object): + "Top-down visitor" + + def visit(self, tree): + return getattr(self, tree.data)(tree) + + def visit_children(self, tree): + return [self.visit(child) if isinstance(child, Tree) else child + for child in tree.children] + + def __getattr__(self, name): + return self.__default__ + + def __default__(self, tree): + return self.visit_children(tree) + + def inline_args(obj): if inspect.isclass(obj) and issubclass(obj, ChildrenTransformer): class _NewTransformer(ChildrenInlineTransformer, obj): diff --git a/tests/test_trees.py b/tests/test_trees.py index 6017386..a41d3de 100644 --- a/tests/test_trees.py +++ b/tests/test_trees.py @@ -5,7 +5,8 @@ from unittest import TestCase import copy import pickle -from lark.tree import Tree, Interpreter, visit_children_decor +from lark.tree import Tree +from lark.transformers import Interpreter, visit_children_decor class TestTrees(TestCase): From 1508dcd7c5f37f62527e9282050df59b237196e7 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sat, 12 May 2018 23:54:22 +0300 Subject: [PATCH 06/12] Refactored inline_args with smart_decorator --- lark/utils.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/lark/utils.py b/lark/utils.py index 0018e49..5e09e7d 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -55,34 +55,34 @@ from contextlib import contextmanager Str = type(u'') -def inline_args(f): - # print '@@', f.__name__, type(f), isinstance(f, types.FunctionType), isinstance(f, types.TypeType), isinstance(f, types.BuiltinFunctionType) +def smart_decorator(f, create_decorator): if isinstance(f, types.FunctionType): - @functools.wraps(f) - def _f_func(self, args): - return f(self, *args) - return _f_func + return functools.wraps(create_decorator(f, True)) + elif isinstance(f, (type, types.BuiltinFunctionType)): - @functools.wraps(f) - def _f_builtin(_self, args): - return f(*args) - return _f_builtin + return functools.wraps(create_decorator(f, False)) + elif isinstance(f, types.MethodType): - @functools.wraps(f.__func__) - def _f(self, args): - return f.__func__(self, *args) - return _f + return functools.wraps(create_decorator(f.__func__, True)) + elif isinstance(f, functools.partial): # wraps does not work for partials in 2.7: https://bugs.python.org/issue3445 - # @functools.wraps(f) - def _f(self, args): - return f(*args) - return _f + return create_decorator(f.__func__, True) + else: - @functools.wraps(f.__call__.__func__) - def _f(self, args): - return f.__call__.__func__(self, *args) - return _f + return create_decorator(f.__func__.__call__, True) + + +def inline_args(f): + def create_decorator(_f, with_self): + if with_self: + def f(self, args): + return _f(self, *args) + else: + def f(args): + return _f(*args) + + return smart_decorator(f, create_decorator) try: From 9daacb9082dea08fee0f81eb94f353fd5af559e2 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sun, 13 May 2018 00:42:50 +0300 Subject: [PATCH 07/12] Refactored transformers, better code --- lark/__init__.py | 1 - lark/load_grammar.py | 31 ++++++------ lark/transformers.py | 113 ++++++++++++++++++++++++++++--------------- lark/tree.py | 2 - lark/utils.py | 20 ++------ tests/test_parser.py | 4 +- 6 files changed, 98 insertions(+), 73 deletions(-) diff --git a/lark/__init__.py b/lark/__init__.py index a2a67b9..5613855 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -3,6 +3,5 @@ from .transformers import Transformer from .common import ParseError, GrammarError, UnexpectedToken from .lexer import UnexpectedInput, LexError from .lark import Lark -from .utils import inline_args __version__ = "0.5.6" diff --git a/lark/load_grammar.py b/lark/load_grammar.py index c710bf8..49b4b8f 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -16,7 +16,7 @@ from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol from .utils import classify, suppress from .tree import Tree, SlottedTree as ST -from .transformers import Transformer, ChildrenTransformer, inline_args, Visitor +from .transformers import Transformer, Visitor, children_args, children_args_inline __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -138,8 +138,8 @@ RULES = { } -@inline_args -class EBNF_to_BNF(ChildrenTransformer): +@children_args_inline +class EBNF_to_BNF(Transformer): def __init__(self): self.new_rules = [] self.rules_by_expr = {} @@ -232,7 +232,8 @@ class SimplifyRule_Visitor(Visitor): tree.children = list(set(tree.children)) -class RuleTreeToText(ChildrenTransformer): +@children_args +class RuleTreeToText(Transformer): def expansions(self, x): return x def expansion(self, symbols): @@ -243,8 +244,8 @@ class RuleTreeToText(ChildrenTransformer): return expansion, alias.value -@inline_args -class CanonizeTree(ChildrenTransformer): +@children_args_inline +class CanonizeTree(Transformer): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) @@ -254,8 +255,8 @@ class CanonizeTree(ChildrenTransformer): tokenmods, value = args return tokenmods + [value] -@inline_args -class PrepareAnonTerminals(ChildrenTransformer): +@children_args_inline +class PrepareAnonTerminals(Transformer): "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them" def __init__(self, tokens): @@ -354,8 +355,8 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -@inline_args -class PrepareLiterals(ChildrenTransformer): +@children_args_inline +class PrepareLiterals(Transformer): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -368,7 +369,8 @@ class PrepareLiterals(ChildrenTransformer): return ST('pattern', [PatternRE(regexp)]) -class TokenTreeToPattern(ChildrenTransformer): +@children_args +class TokenTreeToPattern(Transformer): def pattern(self, ps): p ,= ps return p @@ -407,7 +409,8 @@ class TokenTreeToPattern(ChildrenTransformer): def value(self, v): return v[0] -class PrepareSymbols(ChildrenTransformer): +@children_args +class PrepareSymbols(Transformer): def value(self, v): v ,= v if isinstance(v, Tree): @@ -532,8 +535,8 @@ def options_from_rule(name, *x): def symbols_from_strcase(expansion): return [Terminal(x, filter_out=x.startswith('_')) if is_terminal(x) else NonTerminal(x) for x in expansion] -@inline_args -class PrepareGrammar(ChildrenTransformer): +@children_args_inline +class PrepareGrammar(Transformer): def terminal(self, name): return name def nonterminal(self, name): diff --git a/lark/transformers.py b/lark/transformers.py index d5f13a1..41a3980 100644 --- a/lark/transformers.py +++ b/lark/transformers.py @@ -1,7 +1,7 @@ import inspect from functools import wraps -from . import utils +from .utils import smart_decorator from .tree import Tree class Discard(Exception): @@ -13,46 +13,27 @@ class Base: return getattr(self, tree.data, self.__default__)(tree) def __default__(self, tree): + "Default operation on tree (for override)" return tree class Transformer(Base): def _transform_children(self, children): for c in children: try: - yield self._transform(c) if isinstance(c, Tree) else c + yield self._transform_tree(c) if isinstance(c, Tree) else c except Discard: pass - def _transform(self, tree): + def _transform_tree(self, tree): tree = Tree(tree.data, list(self._transform_children(tree.children))) return self._call_userfunc(tree) def transform(self, tree): - return self._transform(tree) + return self._transform_tree(tree) def __mul__(self, other): return TransformerChain(self, other) -class ChildrenTransformer(Transformer): - def _call_userfunc(self, tree): - # Assumes tree is already transformed - try: - f = getattr(self, tree.data) - except AttributeError: - return self.__default__(tree) - else: - return f(tree.children) - -class ChildrenInlineTransformer(Transformer): - def _call_userfunc(self, tree): - # Assumes tree is already transformed - try: - f = getattr(self, tree.data) - except AttributeError: - return self.__default__(tree) - else: - return f(*tree.children) - class TransformerChain(object): def __init__(self, *transformers): @@ -68,14 +49,22 @@ class TransformerChain(object): class Transformer_InPlace(Transformer): - def _transform(self, tree): + def _transform_tree(self, tree): # Cancel recursion return self._call_userfunc(tree) def transform(self, tree): for subtree in tree.iter_subtrees(): subtree.children = list(self._transform_children(subtree.children)) - return self._transform(tree) + return self._transform_tree(tree) + + +class Transformer_InPlaceRecursive(Transformer): + def _transform_tree(self, tree): + tree.children = list(self._transform_children(tree.children)) + return self._call_userfunc(tree) + + class Visitor(Base): "Bottom-up visitor" @@ -85,11 +74,6 @@ class Visitor(Base): self._call_userfunc(subtree) return tree -class Transformer_InPlaceRecursive(Transformer): - def _transform(self, tree): - tree.children = list(self._transform_children(tree.children)) - return self._call_userfunc(tree) - class Visitor_Recursive(Base): def visit(self, tree): for child in tree.children: @@ -101,7 +85,6 @@ class Visitor_Recursive(Base): return tree -from functools import wraps def visit_children_decor(func): @wraps(func) def inner(cls, tree): @@ -126,11 +109,63 @@ class Interpreter(object): return self.visit_children(tree) -def inline_args(obj): - if inspect.isclass(obj) and issubclass(obj, ChildrenTransformer): - class _NewTransformer(ChildrenInlineTransformer, obj): - pass - return _NewTransformer - else: - return utils.inline_args(obj) +def _children_args__func(f): + @wraps(f) + def create_decorator(_f, with_self): + if with_self: + def f(self, tree): + return _f(self, tree.children) + else: + def f(args): + return _f(tree.children) + + return smart_decorator(f, create_decorator) + +def _children_args__class(cls): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(tree.children) + cls._call_userfunc = _call_userfunc + return cls + + +def children_args(obj): + decorator = _children_args__class if issubclass(obj, Base) else _children_args__func + return decorator(obj) + + + +def _children_args_inline__func(f): + @wraps(f) + def create_decorator(_f, with_self): + if with_self: + def f(self, tree): + return _f(self, *tree.children) + else: + def f(args): + return _f(*tree.children) + + return smart_decorator(f, create_decorator) + + +def _children_args_inline__class(cls): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(*tree.children) + cls._call_userfunc = _call_userfunc + return cls + +def children_args_inline(obj): + decorator = _children_args_inline__class if issubclass(obj, Base) else _children_args_inline__func + return decorator(obj) diff --git a/lark/tree.py b/lark/tree.py index a2b0bef..e20c18d 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -5,8 +5,6 @@ except ImportError: from copy import deepcopy -from .utils import inline_args - class Meta: pass diff --git a/lark/utils.py b/lark/utils.py index 5e09e7d..a4a63dd 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -50,22 +50,22 @@ except NameError: # Python 3 ###{standalone import types -import functools +from functools import wraps, partial from contextlib import contextmanager Str = type(u'') def smart_decorator(f, create_decorator): if isinstance(f, types.FunctionType): - return functools.wraps(create_decorator(f, True)) + return wraps(create_decorator(f, True)) elif isinstance(f, (type, types.BuiltinFunctionType)): - return functools.wraps(create_decorator(f, False)) + return wraps(create_decorator(f, False)) elif isinstance(f, types.MethodType): - return functools.wraps(create_decorator(f.__func__, True)) + return wraps(create_decorator(f.__func__, True)) - elif isinstance(f, functools.partial): + elif isinstance(f, partial): # wraps does not work for partials in 2.7: https://bugs.python.org/issue3445 return create_decorator(f.__func__, True) @@ -73,16 +73,6 @@ def smart_decorator(f, create_decorator): return create_decorator(f.__func__.__call__, True) -def inline_args(f): - def create_decorator(_f, with_self): - if with_self: - def f(self, args): - return _f(self, *args) - else: - def f(args): - return _f(*args) - - return smart_decorator(f, create_decorator) try: diff --git a/tests/test_parser.py b/tests/test_parser.py index e147595..6531cb6 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,8 +21,7 @@ from lark.lark import Lark from lark.common import GrammarError, ParseError, UnexpectedToken from lark.lexer import LexError, UnexpectedInput from lark.tree import Tree -from lark.transformers import ChildrenTransformer as Transformer -# from lark.tree import Transformer +from lark.transformers import Transformer, children_args __path__ = os.path.dirname(__file__) def _read(n, *args): @@ -94,6 +93,7 @@ class TestParsers(unittest.TestCase): self.assertEqual( r.children[0].data, "c" ) def test_embedded_transformer(self): + @children_args class T(Transformer): def a(self, children): return "" From 5e546f38a954a1b8032d7d28ea403e50504e6e05 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sun, 13 May 2018 00:54:06 +0300 Subject: [PATCH 08/12] args decorators actually work now --- lark/__init__.py | 2 +- lark/load_grammar.py | 6 +++--- lark/parsers/earley.py | 2 +- lark/utils.py | 6 +++--- lark/{transformers.py => visitors.py} | 8 +++++--- tests/test_parser.py | 2 +- tests/test_trees.py | 2 +- 7 files changed, 15 insertions(+), 13 deletions(-) rename lark/{transformers.py => visitors.py} (93%) diff --git a/lark/__init__.py b/lark/__init__.py index 5613855..b36b3fc 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,5 +1,5 @@ from .tree import Tree -from .transformers import Transformer +from .visitors import Transformer, Visitor, children_args, children_args_inline from .common import ParseError, GrammarError, UnexpectedToken from .lexer import UnexpectedInput, LexError from .lark import Lark diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 49b4b8f..6262f62 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -16,7 +16,7 @@ from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol from .utils import classify, suppress from .tree import Tree, SlottedTree as ST -from .transformers import Transformer, Visitor, children_args, children_args_inline +from .visitors import Transformer, Visitor, children_args, children_args_inline __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -255,7 +255,6 @@ class CanonizeTree(Transformer): tokenmods, value = args return tokenmods + [value] -@children_args_inline class PrepareAnonTerminals(Transformer): "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them" @@ -266,6 +265,7 @@ class PrepareAnonTerminals(Transformer): self.i = 0 + @children_args_inline def pattern(self, p): value = p.value if p in self.token_reverse and p.flags != self.token_reverse[p].pattern.flags: @@ -409,8 +409,8 @@ class TokenTreeToPattern(Transformer): def value(self, v): return v[0] -@children_args class PrepareSymbols(Transformer): + @children_args def value(self, v): v ,= v if isinstance(v, Tree): diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 5f92fbb..d58b57c 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,7 @@ # Email : erezshin@gmail.com from ..tree import Tree -from ..transformers import Transformer_InPlace +from ..visitors import Transformer_InPlace from ..common import ParseError, UnexpectedToken from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal diff --git a/lark/utils.py b/lark/utils.py index a4a63dd..6f603bb 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -57,13 +57,13 @@ Str = type(u'') def smart_decorator(f, create_decorator): if isinstance(f, types.FunctionType): - return wraps(create_decorator(f, True)) + return wraps(f)(create_decorator(f, True)) elif isinstance(f, (type, types.BuiltinFunctionType)): - return wraps(create_decorator(f, False)) + return wraps(f)(create_decorator(f, False)) elif isinstance(f, types.MethodType): - return wraps(create_decorator(f.__func__, True)) + return wraps(f)(create_decorator(f.__func__, True)) elif isinstance(f, partial): # wraps does not work for partials in 2.7: https://bugs.python.org/issue3445 diff --git a/lark/transformers.py b/lark/visitors.py similarity index 93% rename from lark/transformers.py rename to lark/visitors.py index 41a3980..a1167fa 100644 --- a/lark/transformers.py +++ b/lark/visitors.py @@ -1,4 +1,4 @@ -import inspect +from inspect import isclass from functools import wraps from .utils import smart_decorator @@ -119,6 +119,7 @@ def _children_args__func(f): else: def f(args): return _f(tree.children) + return f return smart_decorator(f, create_decorator) @@ -136,7 +137,7 @@ def _children_args__class(cls): def children_args(obj): - decorator = _children_args__class if issubclass(obj, Base) else _children_args__func + decorator = _children_args__class if isclass(obj) and issubclass(obj, Base) else _children_args__func return decorator(obj) @@ -150,6 +151,7 @@ def _children_args_inline__func(f): else: def f(args): return _f(*tree.children) + return f return smart_decorator(f, create_decorator) @@ -167,5 +169,5 @@ def _children_args_inline__class(cls): return cls def children_args_inline(obj): - decorator = _children_args_inline__class if issubclass(obj, Base) else _children_args_inline__func + decorator = _children_args_inline__class if isclass(obj) and issubclass(obj, Base) else _children_args_inline__func return decorator(obj) diff --git a/tests/test_parser.py b/tests/test_parser.py index 6531cb6..25ce619 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,7 +21,7 @@ from lark.lark import Lark from lark.common import GrammarError, ParseError, UnexpectedToken from lark.lexer import LexError, UnexpectedInput from lark.tree import Tree -from lark.transformers import Transformer, children_args +from lark.visitors import Transformer, children_args __path__ = os.path.dirname(__file__) def _read(n, *args): diff --git a/tests/test_trees.py b/tests/test_trees.py index a41d3de..df3b9b6 100644 --- a/tests/test_trees.py +++ b/tests/test_trees.py @@ -6,7 +6,7 @@ import copy import pickle from lark.tree import Tree -from lark.transformers import Interpreter, visit_children_decor +from lark.visitors import Interpreter, visit_children_decor class TestTrees(TestCase): From 4864a1cf4dedb8b5fc4bea0f9f706b4ebd2d64f5 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Tue, 15 May 2018 10:59:20 +0300 Subject: [PATCH 09/12] More work --- examples/calc.py | 4 +- lark/parsers/earley.py | 7 ++- lark/parsers/resolve_ambig.py | 12 ++--- lark/visitors.py | 92 +++++++++++++++++++++-------------- tests/test_trees.py | 54 +++++++++++++++++++- 5 files changed, 119 insertions(+), 50 deletions(-) diff --git a/examples/calc.py b/examples/calc.py index cb7ef5d..e90b5cc 100644 --- a/examples/calc.py +++ b/examples/calc.py @@ -2,7 +2,7 @@ # This example shows how to write a basic calculator with variables. # -from lark import Lark, InlineTransformer +from lark import Lark, Transformer, children_args_inline try: input = raw_input # For Python2 compatibility @@ -34,7 +34,7 @@ calc_grammar = """ %ignore WS_INLINE """ -class CalculateTree(InlineTransformer): +class CalculateTree(SimpleTransformer): from operator import add, sub, mul, truediv as div, neg number = float diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index d58b57c..87dc41e 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -21,11 +21,10 @@ from ..grammar import NonTerminal class Derivation(Tree): - _hash = None - def __init__(self, rule, items=None): Tree.__init__(self, 'drv', items or []) - self.rule = rule + self.meta.rule = rule + self._hash = None def _pretty_label(self): # Nicer pretty for debugging the parser return self.rule.origin if self.rule else self.data @@ -236,4 +235,4 @@ class ApplyCallbacks(Transformer_InPlace): self.postprocess = postprocess def drv(self, tree): - return self.postprocess[tree.rule](tree.children) + return self.postprocess[tree.meta.rule](tree.children) diff --git a/lark/parsers/resolve_ambig.py b/lark/parsers/resolve_ambig.py index 7c482ae..0d3f17c 100644 --- a/lark/parsers/resolve_ambig.py +++ b/lark/parsers/resolve_ambig.py @@ -16,7 +16,7 @@ def _sum_priority(tree): for n in tree.iter_subtrees(): try: - p += n.rule.options.priority or 0 + p += n.meta.rule.options.priority or 0 except AttributeError: pass @@ -26,8 +26,8 @@ def _compare_priority(tree1, tree2): tree1.iter_subtrees() def _compare_drv(tree1, tree2): - rule1 = getattr(tree1, 'rule', None) - rule2 = getattr(tree2, 'rule', None) + rule1 = getattr(tree1.meta, 'rule', None) + rule2 = getattr(tree2.meta, 'rule', None) if None == rule1 == rule2: return compare(tree1, tree2) @@ -45,7 +45,7 @@ def _compare_drv(tree1, tree2): if c: return c - c = _compare_rules(tree1.rule, tree2.rule) + c = _compare_rules(tree1.meta.rule, tree2.meta.rule) if c: return c @@ -65,7 +65,7 @@ def _standard_resolve_ambig(tree): best = max(tree.children, key=key_f) assert best.data == 'drv' tree.set('drv', best.children) - tree.rule = best.rule # needed for applying callbacks + tree.meta.rule = best.meta.rule # needed for applying callbacks def standard_resolve_ambig(tree): for ambig in tree.find_data('_ambig'): @@ -93,7 +93,7 @@ def _antiscore_sum_resolve_ambig(tree): best = min(tree.children, key=_antiscore_sum_drv) assert best.data == 'drv' tree.set('drv', best.children) - tree.rule = best.rule # needed for applying callbacks + tree.meta.rule = best.meta.rule # needed for applying callbacks def antiscore_sum_resolve_ambig(tree): for ambig in tree.find_data('_ambig'): diff --git a/lark/visitors.py b/lark/visitors.py index a1167fa..d3853bf 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -1,4 +1,4 @@ -from inspect import isclass +from inspect import isclass, getmembers, getmro from functools import wraps from .utils import smart_decorator @@ -16,6 +16,30 @@ class Base: "Default operation on tree (for override)" return tree + @classmethod + def _apply_decorator(cls, decorator): + mro = getmro(cls) + assert mro[0] is cls + libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)} + for name, value in getmembers(cls): + if name.startswith('_') or name in libmembers: + continue + + setattr(cls, name, decorator(value)) + return cls + + +class SimpleBase(Base): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(tree.children) + + class Transformer(Base): def _transform_children(self, children): for c in children: @@ -35,6 +59,7 @@ class Transformer(Base): return TransformerChain(self, other) + class TransformerChain(object): def __init__(self, *transformers): self.transformers = transformers @@ -110,8 +135,22 @@ class Interpreter(object): -def _children_args__func(f): - @wraps(f) + + +def _apply_decorator(obj, decorator): + try: + _apply = obj._apply_decorator + except AttributeError: + return decorator(obj) + else: + return _apply(decorator) + + +def _children_args__func(func): + if getattr(func, '_children_args_decorated', False): + return func + + @wraps(func) def create_decorator(_f, with_self): if with_self: def f(self, tree): @@ -119,55 +158,34 @@ def _children_args__func(f): else: def f(args): return _f(tree.children) + f._children_args_decorated = True return f - return smart_decorator(f, create_decorator) - -def _children_args__class(cls): - def _call_userfunc(self, tree): - # Assumes tree is already transformed - try: - f = getattr(self, tree.data) - except AttributeError: - return self.__default__(tree) - else: - return f(tree.children) - cls._call_userfunc = _call_userfunc - return cls - + return smart_decorator(func, create_decorator) def children_args(obj): - decorator = _children_args__class if isclass(obj) and issubclass(obj, Base) else _children_args__func - return decorator(obj) + return _apply_decorator(obj, _children_args__func) -def _children_args_inline__func(f): - @wraps(f) +def _children_args_inline__func(func): + if getattr(func, '_children_args_decorated', False): + return func + + @wraps(func) def create_decorator(_f, with_self): if with_self: def f(self, tree): return _f(self, *tree.children) else: - def f(args): + def f(self, tree): + print ('##', _f, tree) return _f(*tree.children) + f._children_args_decorated = True return f - return smart_decorator(f, create_decorator) - + return smart_decorator(func, create_decorator) -def _children_args_inline__class(cls): - def _call_userfunc(self, tree): - # Assumes tree is already transformed - try: - f = getattr(self, tree.data) - except AttributeError: - return self.__default__(tree) - else: - return f(*tree.children) - cls._call_userfunc = _call_userfunc - return cls def children_args_inline(obj): - decorator = _children_args_inline__class if isclass(obj) and issubclass(obj, Base) else _children_args_inline__func - return decorator(obj) + return _apply_decorator(obj, _children_args_inline__func) diff --git a/tests/test_trees.py b/tests/test_trees.py index df3b9b6..b7796bf 100644 --- a/tests/test_trees.py +++ b/tests/test_trees.py @@ -6,7 +6,7 @@ import copy import pickle from lark.tree import Tree -from lark.visitors import Interpreter, visit_children_decor +from lark.visitors import Transformer, Interpreter, visit_children_decor, children_args_inline, children_args class TestTrees(TestCase): @@ -59,6 +59,58 @@ class TestTrees(TestCase): self.assertEqual(Interp3().visit(t), list('BCd')) + def test_transformer(self): + t = Tree('add', [Tree('sub', [Tree('i', ['3']), Tree('f', ['1.1'])]), Tree('i', ['1'])]) + + class T(Transformer): + i = children_args_inline(int) + f = children_args_inline(float) + + sub = lambda self, tree: tree.children[0] - tree.children[1] + + def add(self, tree): + return sum(tree.children) + + + res = T().transform(t) + self.assertEqual(res, 2.9) + + @children_args_inline + class T(Transformer): + i = int + f = float + sub = lambda self, a, b: a-b + + def add(self, a, b): + return a + b + + + res = T().transform(t) + self.assertEqual(res, 2.9) + + + @children_args_inline + class T(Transformer): + i = int + f = float + from operator import sub, add + + res = T().transform(t) + self.assertEqual(res, 2.9) + + + @children_args + class T(Transformer): + i = children_args_inline(int) + f = children_args_inline(float) + + sub = lambda self, values: values[0] - values[1] + + def add(self, values): + return sum(values) + + res = T().transform(t) + self.assertEqual(res, 2.9) if __name__ == '__main__': From 6bfc27c11d2d17cc6ea9ee6fab77ff75806b9d87 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Fri, 18 May 2018 15:14:57 +0300 Subject: [PATCH 10/12] New transformers near completion Nearley tool still needs fixing --- examples/calc.py | 5 +- examples/json_parser.py | 6 +- lark/__init__.py | 3 +- lark/load_grammar.py | 15 ++-- lark/parse_tree_builder.py | 16 +++- lark/parsers/earley.py | 11 +-- lark/parsers/resolve_ambig.py | 11 ++- lark/tree.py | 4 +- lark/visitors.py | 145 ++++++++++++++++++++-------------- tests/test_parser.py | 3 +- tests/test_trees.py | 30 ++----- 11 files changed, 140 insertions(+), 109 deletions(-) diff --git a/examples/calc.py b/examples/calc.py index e90b5cc..1102151 100644 --- a/examples/calc.py +++ b/examples/calc.py @@ -2,7 +2,7 @@ # This example shows how to write a basic calculator with variables. # -from lark import Lark, Transformer, children_args_inline +from lark import Lark, Transformer, visitor_args try: input = raw_input # For Python2 compatibility @@ -34,7 +34,8 @@ calc_grammar = """ %ignore WS_INLINE """ -class CalculateTree(SimpleTransformer): +@visitor_args(inline=True) +class CalculateTree(Transformer): from operator import add, sub, mul, truediv as div, neg number = float diff --git a/examples/json_parser.py b/examples/json_parser.py index 38a5b70..216af2c 100644 --- a/examples/json_parser.py +++ b/examples/json_parser.py @@ -7,7 +7,7 @@ import sys -from lark import Lark, inline_args, Transformer +from lark import Lark, Transformer, visitor_args json_grammar = r""" ?start: value @@ -34,14 +34,14 @@ json_grammar = r""" """ class TreeToJson(Transformer): - @inline_args + @visitor_args(inline=True) def string(self, s): return s[1:-1].replace('\\"', '"') array = list pair = tuple object = dict - number = inline_args(float) + number = visitor_args(inline=True)(float) null = lambda self, _: None true = lambda self, _: True diff --git a/lark/__init__.py b/lark/__init__.py index b36b3fc..850f702 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,5 +1,6 @@ from .tree import Tree -from .visitors import Transformer, Visitor, children_args, children_args_inline +from .visitors import Transformer, Visitor, visitor_args, Discard +from .visitors import InlineTransformer, inline_args # XXX Deprecated from .common import ParseError, GrammarError, UnexpectedToken from .lexer import UnexpectedInput, LexError from .lark import Lark diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 6262f62..5cf8e19 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -16,7 +16,7 @@ from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol from .utils import classify, suppress from .tree import Tree, SlottedTree as ST -from .visitors import Transformer, Visitor, children_args, children_args_inline +from .visitors import Transformer, Visitor, visitor_args __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -138,7 +138,7 @@ RULES = { } -@children_args_inline +@visitor_args(inline=True) class EBNF_to_BNF(Transformer): def __init__(self): self.new_rules = [] @@ -232,7 +232,6 @@ class SimplifyRule_Visitor(Visitor): tree.children = list(set(tree.children)) -@children_args class RuleTreeToText(Transformer): def expansions(self, x): return x @@ -244,7 +243,7 @@ class RuleTreeToText(Transformer): return expansion, alias.value -@children_args_inline +@visitor_args(inline=True) class CanonizeTree(Transformer): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) @@ -265,7 +264,7 @@ class PrepareAnonTerminals(Transformer): self.i = 0 - @children_args_inline + @visitor_args(inline=True) def pattern(self, p): value = p.value if p in self.token_reverse and p.flags != self.token_reverse[p].pattern.flags: @@ -355,7 +354,7 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -@children_args_inline +@visitor_args(inline=True) class PrepareLiterals(Transformer): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -369,7 +368,6 @@ class PrepareLiterals(Transformer): return ST('pattern', [PatternRE(regexp)]) -@children_args class TokenTreeToPattern(Transformer): def pattern(self, ps): p ,= ps @@ -410,7 +408,6 @@ class TokenTreeToPattern(Transformer): return v[0] class PrepareSymbols(Transformer): - @children_args def value(self, v): v ,= v if isinstance(v, Tree): @@ -535,7 +532,7 @@ def options_from_rule(name, *x): def symbols_from_strcase(expansion): return [Terminal(x, filter_out=x.startswith('_')) if is_terminal(x) else NonTerminal(x) for x in expansion] -@children_args_inline +@visitor_args(inline=True) class PrepareGrammar(Transformer): def terminal(self, name): return name diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 7c624e2..abca756 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -3,9 +3,10 @@ from .utils import suppress from .lexer import Token from .grammar import Rule from .tree import Tree +from .visitors import InlineTransformer # XXX Deprecated ###{standalone -from functools import partial +from functools import partial, wraps class ExpandSingleChild: @@ -95,6 +96,15 @@ def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous): class Callback(object): pass + +def inline_args(func): + @wraps(func) + def f(children): + return func(*children) + return f + + + class ParseTreeBuilder: def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False): self.tree_class = tree_class @@ -130,6 +140,10 @@ class ParseTreeBuilder: user_callback_name = rule.alias or rule.origin.name try: f = getattr(transformer, user_callback_name) + assert not getattr(f, 'meta', False), "Meta args not supported for internal transformer" + # XXX InlineTransformer is deprecated! + if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer): + f = inline_args(f) except AttributeError: f = partial(self.tree_class, user_callback_name) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 87dc41e..46a1271 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,7 @@ # Email : erezshin@gmail.com from ..tree import Tree -from ..visitors import Transformer_InPlace +from ..visitors import Transformer_InPlace, visitor_args from ..common import ParseError, UnexpectedToken from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal @@ -114,9 +114,9 @@ class Column: if old_tree.data != '_ambig': new_tree = old_tree.copy() - new_tree.rule = old_tree.rule + new_tree.meta.rule = old_tree.meta.rule old_tree.set('_ambig', [new_tree]) - old_tree.rule = None # No longer a 'drv' node + old_tree.meta.rule = None # No longer a 'drv' node if item.tree.children[0] is old_tree: # XXX a little hacky! raise ParseError("Infinite recursion in grammar! (Rule %s)" % item.rule) @@ -234,5 +234,6 @@ class ApplyCallbacks(Transformer_InPlace): def __init__(self, postprocess): self.postprocess = postprocess - def drv(self, tree): - return self.postprocess[tree.meta.rule](tree.children) + @visitor_args(meta=True) + def drv(self, children, meta): + return self.postprocess[meta.rule](children) diff --git a/lark/parsers/resolve_ambig.py b/lark/parsers/resolve_ambig.py index 0d3f17c..2470eb9 100644 --- a/lark/parsers/resolve_ambig.py +++ b/lark/parsers/resolve_ambig.py @@ -26,8 +26,15 @@ def _compare_priority(tree1, tree2): tree1.iter_subtrees() def _compare_drv(tree1, tree2): - rule1 = getattr(tree1.meta, 'rule', None) - rule2 = getattr(tree2.meta, 'rule', None) + try: + rule1 = tree1.meta.rule + except AttributeError: + rule1 = None + + try: + rule2 = tree2.meta.rule + except AttributeError: + rule2 = None if None == rule1 == rule2: return compare(tree1, tree2) diff --git a/lark/tree.py b/lark/tree.py index e20c18d..1490632 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -10,10 +10,10 @@ class Meta: ###{standalone class Tree(object): - def __init__(self, data, children): + def __init__(self, data, children, meta=None): self.data = data self.children = children - self._meta = None + self._meta = meta @property def meta(self): diff --git a/lark/visitors.py b/lark/visitors.py index d3853bf..950371f 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -8,39 +8,23 @@ class Discard(Exception): pass -class Base: - def _call_userfunc(self, tree): - return getattr(self, tree.data, self.__default__)(tree) +# Transformers - def __default__(self, tree): - "Default operation on tree (for override)" - return tree - - @classmethod - def _apply_decorator(cls, decorator): - mro = getmro(cls) - assert mro[0] is cls - libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)} - for name, value in getmembers(cls): - if name.startswith('_') or name in libmembers: - continue - - setattr(cls, name, decorator(value)) - return cls - - -class SimpleBase(Base): - def _call_userfunc(self, tree): +class Transformer: + def _call_userfunc(self, data, children, meta): # Assumes tree is already transformed try: - f = getattr(self, tree.data) + f = getattr(self, data) except AttributeError: - return self.__default__(tree) + return self.__default__(data, children, meta) else: - return f(tree.children) - + if getattr(f, 'meta', False): + return f(children, meta) + elif getattr(f, 'inline', False): + return f(*children) + else: + return f(children) -class Transformer(Base): def _transform_children(self, children): for c in children: try: @@ -49,8 +33,8 @@ class Transformer(Base): pass def _transform_tree(self, tree): - tree = Tree(tree.data, list(self._transform_children(tree.children))) - return self._call_userfunc(tree) + children = list(self._transform_children(tree.children)) + return self._call_userfunc(tree.data, children, tree.meta) def transform(self, tree): return self._transform_tree(tree) @@ -58,6 +42,32 @@ class Transformer(Base): def __mul__(self, other): return TransformerChain(self, other) + def __default__(self, data, children, meta): + "Default operation on tree (for override)" + return Tree(data, children, meta) + + @classmethod + def _apply_decorator(cls, decorator, **kwargs): + mro = getmro(cls) + assert mro[0] is cls + libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)} + for name, value in getmembers(cls): + if name.startswith('_') or name in libmembers: + continue + + setattr(cls, name, decorator(value, **kwargs)) + return cls + + +class InlineTransformer(Transformer): # XXX Deprecated + def _call_userfunc(self, data, children, meta): + # Assumes tree is already transformed + try: + f = getattr(self, data) + except AttributeError: + return self.__default__(data, children, meta) + else: + return f(*children) class TransformerChain(object): @@ -75,7 +85,7 @@ class TransformerChain(object): class Transformer_InPlace(Transformer): def _transform_tree(self, tree): # Cancel recursion - return self._call_userfunc(tree) + return self._call_userfunc(tree.data, tree.children, tree.meta) def transform(self, tree): for subtree in tree.iter_subtrees(): @@ -87,11 +97,22 @@ class Transformer_InPlace(Transformer): class Transformer_InPlaceRecursive(Transformer): def _transform_tree(self, tree): tree.children = list(self._transform_children(tree.children)) - return self._call_userfunc(tree) + return self._call_userfunc(tree.data, tree.children, tree.meta) + +# Visitors -class Visitor(Base): +class VisitorBase: + def _call_userfunc(self, tree): + return getattr(self, tree.data, self.__default__)(tree) + + def __default__(self, tree): + "Default operation on tree (for override)" + return tree + + +class Visitor(VisitorBase): "Bottom-up visitor" def visit(self, tree): @@ -99,7 +120,7 @@ class Visitor(Base): self._call_userfunc(subtree) return tree -class Visitor_Recursive(Base): +class Visitor_Recursive(VisitorBase): def visit(self, tree): for child in tree.children: if isinstance(child, Tree): @@ -110,6 +131,7 @@ class Visitor_Recursive(Base): return tree + def visit_children_decor(func): @wraps(func) def inner(cls, tree): @@ -117,7 +139,8 @@ def visit_children_decor(func): return func(cls, values) return inner -class Interpreter(object): + +class Interpreter: "Top-down visitor" def visit(self, tree): @@ -136,56 +159,58 @@ class Interpreter(object): +# Decorators -def _apply_decorator(obj, decorator): +def _apply_decorator(obj, decorator, **kwargs): try: _apply = obj._apply_decorator except AttributeError: - return decorator(obj) + return decorator(obj, **kwargs) else: - return _apply(decorator) + return _apply(decorator, **kwargs) -def _children_args__func(func): - if getattr(func, '_children_args_decorated', False): - return func +def _inline_args__func(func): @wraps(func) def create_decorator(_f, with_self): if with_self: - def f(self, tree): - return _f(self, tree.children) + def f(self, children): + return _f(self, *children) else: - def f(args): - return _f(tree.children) - f._children_args_decorated = True + def f(self, children): + return _f(*children) return f return smart_decorator(func, create_decorator) -def children_args(obj): - return _apply_decorator(obj, _children_args__func) +def inline_args(obj): # XXX Deprecated + return _apply_decorator(obj, _inline_args__func) -def _children_args_inline__func(func): - if getattr(func, '_children_args_decorated', False): - return func - @wraps(func) +def _visitor_args_func_dec(func, inline=False, meta=False): + assert not (inline and meta) def create_decorator(_f, with_self): if with_self: - def f(self, tree): - return _f(self, *tree.children) + def f(self, *args, **kwargs): + return _f(self, *args, **kwargs) else: - def f(self, tree): - print ('##', _f, tree) - return _f(*tree.children) - f._children_args_decorated = True + def f(self, *args, **kwargs): + return _f(*args, **kwargs) return f - return smart_decorator(func, create_decorator) + f = smart_decorator(func, create_decorator) + f.inline = inline + f.meta = meta + return f + +def visitor_args(inline=False, meta=False): + if inline and meta: + raise ValueError("Visitor functions can either accept meta, or be inlined. Not both.") + def _visitor_args_dec(obj): + return _apply_decorator(obj, _visitor_args_func_dec, inline=inline, meta=meta) + return _visitor_args_dec -def children_args_inline(obj): - return _apply_decorator(obj, _children_args_inline__func) diff --git a/tests/test_parser.py b/tests/test_parser.py index 25ce619..f48f3bd 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,7 +21,7 @@ from lark.lark import Lark from lark.common import GrammarError, ParseError, UnexpectedToken from lark.lexer import LexError, UnexpectedInput from lark.tree import Tree -from lark.visitors import Transformer, children_args +from lark.visitors import Transformer __path__ = os.path.dirname(__file__) def _read(n, *args): @@ -93,7 +93,6 @@ class TestParsers(unittest.TestCase): self.assertEqual( r.children[0].data, "c" ) def test_embedded_transformer(self): - @children_args class T(Transformer): def a(self, children): return "" diff --git a/tests/test_trees.py b/tests/test_trees.py index b7796bf..af5a2a0 100644 --- a/tests/test_trees.py +++ b/tests/test_trees.py @@ -6,7 +6,7 @@ import copy import pickle from lark.tree import Tree -from lark.visitors import Transformer, Interpreter, visit_children_decor, children_args_inline, children_args +from lark.visitors import Transformer, Interpreter, visit_children_decor, visitor_args class TestTrees(TestCase): @@ -63,19 +63,18 @@ class TestTrees(TestCase): t = Tree('add', [Tree('sub', [Tree('i', ['3']), Tree('f', ['1.1'])]), Tree('i', ['1'])]) class T(Transformer): - i = children_args_inline(int) - f = children_args_inline(float) + i = visitor_args(inline=True)(int) + f = visitor_args(inline=True)(float) - sub = lambda self, tree: tree.children[0] - tree.children[1] - - def add(self, tree): - return sum(tree.children) + sub = lambda self, values: values[0] - values[1] + def add(self, values): + return sum(values) res = T().transform(t) self.assertEqual(res, 2.9) - @children_args_inline + @visitor_args(inline=True) class T(Transformer): i = int f = float @@ -89,7 +88,7 @@ class TestTrees(TestCase): self.assertEqual(res, 2.9) - @children_args_inline + @visitor_args(inline=True) class T(Transformer): i = int f = float @@ -99,19 +98,6 @@ class TestTrees(TestCase): self.assertEqual(res, 2.9) - @children_args - class T(Transformer): - i = children_args_inline(int) - f = children_args_inline(float) - - sub = lambda self, values: values[0] - values[1] - - def add(self, values): - return sum(values) - - res = T().transform(t) - self.assertEqual(res, 2.9) - if __name__ == '__main__': unittest.main() From 6d76a4ce8de0231cca71b22b7a4d1acf5ad7aa4f Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Mon, 18 Jun 2018 11:30:32 +0300 Subject: [PATCH 11/12] visitor_args -> v_args --- examples/calc.py | 4 ++-- examples/json_parser.py | 6 +++--- lark/__init__.py | 2 +- lark/load_grammar.py | 13 +++++++------ lark/parsers/earley.py | 4 ++-- lark/visitors.py | 2 +- tests/test_trees.py | 10 +++++----- 7 files changed, 21 insertions(+), 20 deletions(-) diff --git a/examples/calc.py b/examples/calc.py index 1102151..a187571 100644 --- a/examples/calc.py +++ b/examples/calc.py @@ -2,7 +2,7 @@ # This example shows how to write a basic calculator with variables. # -from lark import Lark, Transformer, visitor_args +from lark import Lark, Transformer, v_args try: input = raw_input # For Python2 compatibility @@ -34,7 +34,7 @@ calc_grammar = """ %ignore WS_INLINE """ -@visitor_args(inline=True) +@v_args(inline=True) class CalculateTree(Transformer): from operator import add, sub, mul, truediv as div, neg number = float diff --git a/examples/json_parser.py b/examples/json_parser.py index 216af2c..a0c2d3b 100644 --- a/examples/json_parser.py +++ b/examples/json_parser.py @@ -7,7 +7,7 @@ import sys -from lark import Lark, Transformer, visitor_args +from lark import Lark, Transformer, v_args json_grammar = r""" ?start: value @@ -34,14 +34,14 @@ json_grammar = r""" """ class TreeToJson(Transformer): - @visitor_args(inline=True) + @v_args(inline=True) def string(self, s): return s[1:-1].replace('\\"', '"') array = list pair = tuple object = dict - number = visitor_args(inline=True)(float) + number = v_args(inline=True)(float) null = lambda self, _: None true = lambda self, _: True diff --git a/lark/__init__.py b/lark/__init__.py index 850f702..0146664 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,5 +1,5 @@ from .tree import Tree -from .visitors import Transformer, Visitor, visitor_args, Discard +from .visitors import Transformer, Visitor, v_args, Discard from .visitors import InlineTransformer, inline_args # XXX Deprecated from .common import ParseError, GrammarError, UnexpectedToken from .lexer import UnexpectedInput, LexError diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 5cf8e19..42b27df 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -16,7 +16,8 @@ from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol from .utils import classify, suppress from .tree import Tree, SlottedTree as ST -from .visitors import Transformer, Visitor, visitor_args +from .visitors import Transformer, Visitor, v_args +inline_args = v_args(inline=True) __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -138,7 +139,7 @@ RULES = { } -@visitor_args(inline=True) +@inline_args class EBNF_to_BNF(Transformer): def __init__(self): self.new_rules = [] @@ -243,7 +244,7 @@ class RuleTreeToText(Transformer): return expansion, alias.value -@visitor_args(inline=True) +@inline_args class CanonizeTree(Transformer): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) @@ -264,7 +265,7 @@ class PrepareAnonTerminals(Transformer): self.i = 0 - @visitor_args(inline=True) + @inline_args def pattern(self, p): value = p.value if p in self.token_reverse and p.flags != self.token_reverse[p].pattern.flags: @@ -354,7 +355,7 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -@visitor_args(inline=True) +@inline_args class PrepareLiterals(Transformer): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -532,7 +533,7 @@ def options_from_rule(name, *x): def symbols_from_strcase(expansion): return [Terminal(x, filter_out=x.startswith('_')) if is_terminal(x) else NonTerminal(x) for x in expansion] -@visitor_args(inline=True) +@inline_args class PrepareGrammar(Transformer): def terminal(self, name): return name diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 46a1271..65e0ea5 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,7 @@ # Email : erezshin@gmail.com from ..tree import Tree -from ..visitors import Transformer_InPlace, visitor_args +from ..visitors import Transformer_InPlace, v_args from ..common import ParseError, UnexpectedToken from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal @@ -234,6 +234,6 @@ class ApplyCallbacks(Transformer_InPlace): def __init__(self, postprocess): self.postprocess = postprocess - @visitor_args(meta=True) + @v_args(meta=True) def drv(self, children, meta): return self.postprocess[meta.rule](children) diff --git a/lark/visitors.py b/lark/visitors.py index 950371f..2f87c25 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -206,7 +206,7 @@ def _visitor_args_func_dec(func, inline=False, meta=False): f.meta = meta return f -def visitor_args(inline=False, meta=False): +def v_args(inline=False, meta=False): if inline and meta: raise ValueError("Visitor functions can either accept meta, or be inlined. Not both.") def _visitor_args_dec(obj): diff --git a/tests/test_trees.py b/tests/test_trees.py index af5a2a0..1debd89 100644 --- a/tests/test_trees.py +++ b/tests/test_trees.py @@ -6,7 +6,7 @@ import copy import pickle from lark.tree import Tree -from lark.visitors import Transformer, Interpreter, visit_children_decor, visitor_args +from lark.visitors import Transformer, Interpreter, visit_children_decor, v_args class TestTrees(TestCase): @@ -63,8 +63,8 @@ class TestTrees(TestCase): t = Tree('add', [Tree('sub', [Tree('i', ['3']), Tree('f', ['1.1'])]), Tree('i', ['1'])]) class T(Transformer): - i = visitor_args(inline=True)(int) - f = visitor_args(inline=True)(float) + i = v_args(inline=True)(int) + f = v_args(inline=True)(float) sub = lambda self, values: values[0] - values[1] @@ -74,7 +74,7 @@ class TestTrees(TestCase): res = T().transform(t) self.assertEqual(res, 2.9) - @visitor_args(inline=True) + @v_args(inline=True) class T(Transformer): i = int f = float @@ -88,7 +88,7 @@ class TestTrees(TestCase): self.assertEqual(res, 2.9) - @visitor_args(inline=True) + @v_args(inline=True) class T(Transformer): i = int f = float From c77934f6a24e28ae5c1716fa90c47ce46235fd8d Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Mon, 18 Jun 2018 11:32:07 +0300 Subject: [PATCH 12/12] Fixed nearley --- lark/tools/nearley.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lark/tools/nearley.py b/lark/tools/nearley.py index eead26d..a7fd259 100644 --- a/lark/tools/nearley.py +++ b/lark/tools/nearley.py @@ -160,7 +160,7 @@ def create_code_for_nearley_grammar(g, start, builtin_path, folder_path): emit('class TransformNearley(Transformer):') for alias in n2l.alias_js_code: emit(" %s = var.get('%s').to_python()" % (alias, alias)) - emit(" __default__ = lambda self, n, c: c if c else None") + emit(" __default__ = lambda self, n, c, m: c if c else None") emit() emit('parser = Lark(grammar, start="n_%s")' % start)