From b2489e13e2de3df45557d79873ee91243cdf958a Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Wed, 14 Nov 2018 00:44:50 +0200 Subject: [PATCH] A few fixes (tests + interface) --- lark/parser_frontends.py | 2 +- lark/parsers/earley.py | 10 ++++------ lark/parsers/xearley.py | 7 +++---- tests/__main__.py | 1 + tests/test_parser.py | 21 +++++++++------------ 5 files changed, 18 insertions(+), 23 deletions(-) diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index c60898e..160cc4d 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -118,7 +118,7 @@ class XEarley: class XEarley_CompleteLex(XEarley): def __init__(self, *args, **kw): - super(self).__init__(*args, complete_lex=True, **kw) + XEarley.__init__(self, *args, complete_lex=True, **kw) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 7dfa26c..c514f13 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -9,17 +9,15 @@ The Earley parser outputs an SPPF-tree as per that document. The SPPF tree forma is better documented here: http://www.bramvandersanden.com/post/2014/06/shared-packed-parse-forest/ """ -# Author: Erez Shinan (2017) -# Email : erezshin@gmail.com + +from collections import deque, defaultdict from ..visitors import Transformer_InPlace, v_args from ..exceptions import ParseError, UnexpectedToken from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal from .earley_common import Column, Item -from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode - -from collections import deque, defaultdict +from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode, ForestToAmbiguousTreeVisitor class Parser: def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, forest_sum_visitor = ForestSumVisitor): @@ -199,7 +197,7 @@ class Parser: ## If we're not resolving ambiguity, we just return the root of the SPPF tree to the caller. # This means the caller can work directly with the SPPF tree. if not self.resolve_ambiguity: - return solutions[0] + return ForestToAmbiguousTreeVisitor(solutions[0], self.callbacks).go() # ... otherwise, disambiguate and convert the SPPF to an AST, removing any ambiguities # according to the rules. diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py index 4bdccc4..b42a305 100644 --- a/lark/parsers/xearley.py +++ b/lark/parsers/xearley.py @@ -20,12 +20,10 @@ from collections import defaultdict, deque from ..exceptions import ParseError, UnexpectedCharacters from ..lexer import Token -from ..tree import Tree from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal, Terminal -from .earley import ApplyCallbacks from .earley_common import Column, Item -from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode +from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode, ForestToAmbiguousTreeVisitor class Parser: @@ -267,8 +265,9 @@ class Parser: ## If we're not resolving ambiguity, we just return the root of the SPPF tree to the caller. # This means the caller can work directly with the SPPF tree. if not self.resolve_ambiguity: - return solutions[0] + return ForestToAmbiguousTreeVisitor(solutions[0], self.callbacks).go() # ... otherwise, disambiguate and convert the SPPF to an AST, removing any ambiguities # according to the rules. return ForestToTreeVisitor(solutions[0], self.forest_sum_visitor, self.callbacks).go() + diff --git a/tests/__main__.py b/tests/__main__.py index 051ead6..51a3f67 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -25,6 +25,7 @@ from .test_parser import ( # TestFullEarleyScanless, TestFullEarleyDynamic, + TestFullEarleyDynamic_complete, TestParsers, ) diff --git a/tests/test_parser.py b/tests/test_parser.py index a54ecc4..b5b5519 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,7 +21,6 @@ from lark.lark import Lark from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput from lark.tree import Tree from lark.visitors import Transformer -from lark.parsers.earley_forest import ForestToAmbiguousTreeVisitor from lark.parsers.earley import ApplyCallbacks __path__ = os.path.dirname(__file__) @@ -156,6 +155,8 @@ class TestParsers(unittest.TestCase): def _make_full_earley_test(LEXER): + def _Lark(grammar, **kwargs): + return Lark(grammar, lexer=LEXER, parser='earley', propagate_positions=True, **kwargs) class _TestFullEarley(unittest.TestCase): def test_anon(self): # Fails an Earley implementation without special handling for empty rules, @@ -238,8 +239,7 @@ def _make_full_earley_test(LEXER): """ parser = Lark(grammar, parser='earley', lexer=LEXER, ambiguity='explicit') - root_symbol = parser.parse('ab') - ambig_tree = ForestToAmbiguousTreeVisitor(root_symbol, parser.parser.parser.callbacks).go() + ambig_tree = parser.parse('ab') # print(ambig_tree.pretty()) self.assertEqual( ambig_tree.data, '_ambig') self.assertEqual( len(ambig_tree.children), 2) @@ -254,8 +254,7 @@ def _make_full_earley_test(LEXER): """ l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER) - root_symbol = l.parse('cde') - ambig_tree = ForestToAmbiguousTreeVisitor(root_symbol, l.parser.parser.callbacks).go() + ambig_tree = l.parse('cde') # print(ambig_tree.pretty()) # tree = ApplyCallbacks(l.parser.parser.postprocess).transform(ambig_tree) @@ -301,8 +300,7 @@ def _make_full_earley_test(LEXER): %ignore WS """ parser = Lark(grammar, ambiguity='explicit', lexer=LEXER) - root_symbol = parser.parse('fruit flies like bananas') - tree = ForestToAmbiguousTreeVisitor(root_symbol, parser.parser.parser.callbacks).go() + tree = parser.parse('fruit flies like bananas') # tree = ApplyCallbacks(parser.parser.parser.postprocess).transform(ambig_tree) expected = Tree('_ambig', [ @@ -335,10 +333,9 @@ def _make_full_earley_test(LEXER): """ text = """cat""" - parser = Lark(grammar, start='start', ambiguity='explicit') - root_symbol = parser.parse(text) - ambig_tree = ForestToAmbiguousTreeVisitor(root_symbol).go() - tree = ApplyCallbacks(parser.parser.parser.postprocess).transform(ambig_tree) + parser = _Lark(grammar, start='start', ambiguity='explicit') + tree = parser.parse(text) + print(tree.pretty()) self.assertEqual(tree.data, '_ambig') combinations = {tuple(str(s) for s in t.children) for t in tree.children} @@ -1303,7 +1300,7 @@ _TO_TEST = [ for _LEXER, _PARSER in _TO_TEST: _make_parser_test(_LEXER, _PARSER) -for _LEXER in ('dynamic',): +for _LEXER in ('dynamic', 'dynamic_complete'): _make_full_earley_test(_LEXER) if __name__ == '__main__':