diff --git a/examples/lark_grammar.py b/examples/lark_grammar.py index f2185ee..6730db9 100644 --- a/examples/lark_grammar.py +++ b/examples/lark_grammar.py @@ -7,13 +7,13 @@ A reference implementation of the Lark grammar (using LALR(1)) import lark from pathlib import Path -parser = lark.Lark.open('lark.lark', rel_to=__file__, parser="lalr") - examples_path = Path(__file__).parent lark_path = Path(lark.__file__).parent +parser = lark.Lark.open(lark_path / 'grammars/lark.lark', rel_to=__file__, parser="lalr") + + grammar_files = [ - examples_path / 'lark.lark', examples_path / 'advanced/python2.lark', examples_path / 'advanced/python3.lark', examples_path / 'relative-imports/multiples.lark', @@ -21,7 +21,11 @@ grammar_files = [ examples_path / 'relative-imports/multiple3.lark', examples_path / 'tests/no_newline_at_end.lark', examples_path / 'tests/negative_priority.lark', + examples_path / 'standalone/json.lark', lark_path / 'grammars/common.lark', + lark_path / 'grammars/lark.lark', + lark_path / 'grammars/unicode.lark', + lark_path / 'grammars/python.lark', ] def test(): diff --git a/examples/lark.lark b/lark/grammars/lark.lark similarity index 96% rename from examples/lark.lark rename to lark/grammars/lark.lark index 474b819..c16db47 100644 --- a/examples/lark.lark +++ b/lark/grammars/lark.lark @@ -45,7 +45,7 @@ OP: /[+*]|[?](?![a-z])/ RULE: /!?[_?]?[a-z][_a-z0-9]*/ TOKEN: /_?[A-Z][_A-Z0-9]*/ STRING: _STRING "i"? -REGEXP: /\/(?!\/)(\\\/|\\\\|[^\/\n])*?\/[imslux]*/ +REGEXP: /\/(?!\/)(\\\/|\\\\|[^\/])*?\/[imslux]*/ _NL: /(\r?\n)+\s*/ %import common.ESCAPED_STRING -> _STRING diff --git a/tests/test_parser.py b/tests/test_parser.py index d8f1e48..86a6be1 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -30,6 +30,7 @@ try: except ImportError: regex = None +import lark from lark import logger from lark.lark import Lark from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters @@ -866,9 +867,9 @@ class CustomLexerNew(Lexer): self.lexer = TraditionalLexer(copy(lexer_conf)) def lex(self, lexer_state, parser_state): return self.lexer.lex(lexer_state, parser_state) - + __future_interface__ = True - + class CustomLexerOld(Lexer): """ Purpose of this custom lexer is to test the integration, @@ -879,7 +880,7 @@ class CustomLexerOld(Lexer): def lex(self, text): ls = self.lexer.make_lexer_state(text) return self.lexer.lex(ls, None) - + __future_interface__ = False def _tree_structure_check(a, b): @@ -954,17 +955,30 @@ class DualBytesLark: self.bytes_lark.load(f) def _make_parser_test(LEXER, PARSER): - if LEXER == 'custom_new': - lexer_class_or_name = CustomLexerNew - elif LEXER == 'custom_old': - lexer_class_or_name = CustomLexerOld - else: - lexer_class_or_name = LEXER + lexer_class_or_name = { + 'custom_new': CustomLexerNew, + 'custom_old': CustomLexerOld, + }.get(LEXER, LEXER) + def _Lark(grammar, **kwargs): return Lark(grammar, lexer=lexer_class_or_name, parser=PARSER, propagate_positions=True, **kwargs) def _Lark_open(gfilename, **kwargs): return Lark.open(gfilename, lexer=lexer_class_or_name, parser=PARSER, propagate_positions=True, **kwargs) + if (LEXER, PARSER) == ('standard', 'earley'): + # Check that the `lark.lark` grammar represents can parse every example used in these tests. + # Standard-Earley was an arbitrary choice, to make sure it only ran once. + lalr_parser = Lark.open(os.path.join(os.path.dirname(lark.__file__), 'grammars/lark.lark'), parser='lalr') + def wrap_with_test_grammar(f): + def _f(x, **kwargs): + inst = f(x, **kwargs) + lalr_parser.parse(inst.source_grammar) # Test after instance creation. When the grammar should fail, don't test it. + return inst + return _f + + _Lark = wrap_with_test_grammar(_Lark) + _Lark_open = wrap_with_test_grammar(_Lark_open) + class _TestParser(unittest.TestCase): def test_basic1(self): @@ -2450,12 +2464,12 @@ _TO_TEST = [ ('standard', 'earley'), ('standard', 'cyk'), ('standard', 'lalr'), - + ('dynamic', 'earley'), ('dynamic_complete', 'earley'), - + ('contextual', 'lalr'), - + ('custom_new', 'lalr'), ('custom_old', 'earley'), ]