diff --git a/docs/grammar.md b/docs/grammar.md index 506e93f..d8aeb07 100644 --- a/docs/grammar.md +++ b/docs/grammar.md @@ -133,20 +133,29 @@ When importing rules, all their dependencies will be imported into a namespace, **Syntax:** ```html %import . -%import ( ) +%import . +%import . -> +%import . -> +%import ( ) ``` If the module path is absolute, Lark will attempt to load it from the built-in directory (currently, only `common.lark` is available). If the module path is relative, such as `.path.to.file`, Lark will attempt to load it from the current working directory. Grammars must have the `.lark` extension. +The rule or terminal can be imported under an other name with the `->` syntax. + **Example:** ```perl %import common.NUMBER %import .terminals_file (A B C) + +%import .rules_file.rulea -> ruleb ``` +Note that `%ignore` directives cannot be imported. Imported rules will abide by the `%ignore` directives declared in the main grammar. + ### %declare Declare a terminal without defining it. Useful for plugins. diff --git a/examples/lark.lark b/examples/lark.lark index e561dbe..915cf2e 100644 --- a/examples/lark.lark +++ b/examples/lark.lark @@ -10,10 +10,10 @@ token: TOKEN priority? ":" expansions _NL priority: "." NUMBER statement: "%ignore" expansions _NL -> ignore - | "%import" import_args ["->" TOKEN] _NL -> import + | "%import" import_args ["->" name] _NL -> import | "%declare" name+ -> declare -import_args: name ("." name)* +import_args: "."? name ("." name)* ?expansions: alias (_VBAR alias)* diff --git a/examples/lark_grammar.py b/examples/lark_grammar.py index 30ccc8b..c7ace47 100644 --- a/examples/lark_grammar.py +++ b/examples/lark_grammar.py @@ -6,6 +6,9 @@ grammar_files = [ 'examples/python2.lark', 'examples/python3.lark', 'examples/lark.lark', + 'examples/relative-imports/multiples.lark', + 'examples/relative-imports/multiple2.lark', + 'examples/relative-imports/multiple3.lark', 'lark/grammars/common.lark', ] diff --git a/examples/relative-imports/multiple2.lark b/examples/relative-imports/multiple2.lark new file mode 100644 index 0000000..a65077c --- /dev/null +++ b/examples/relative-imports/multiple2.lark @@ -0,0 +1 @@ +start: ("0" | "1")* "0" diff --git a/examples/relative-imports/multiple3.lark b/examples/relative-imports/multiple3.lark new file mode 100644 index 0000000..6a67bce --- /dev/null +++ b/examples/relative-imports/multiple3.lark @@ -0,0 +1,5 @@ +start: mod0mod0+ + +mod0mod0: "0" | "1" mod1mod0 +mod1mod0: "1" | "0" mod2mod1 mod1mod0 +mod2mod1: "0" | "1" mod2mod1 diff --git a/examples/relative-imports/multiples.lark b/examples/relative-imports/multiples.lark new file mode 100644 index 0000000..35b5d17 --- /dev/null +++ b/examples/relative-imports/multiples.lark @@ -0,0 +1,5 @@ +start: "2:" multiple2 + | "3:" multiple3 + +%import .multiple2.start -> multiple2 +%import .multiple3.start -> multiple3 diff --git a/examples/relative-imports/multiples.py b/examples/relative-imports/multiples.py new file mode 100644 index 0000000..b57d446 --- /dev/null +++ b/examples/relative-imports/multiples.py @@ -0,0 +1,28 @@ +# +# This example demonstrates relative imports with rule rewrite +# see multiples.lark +# + +# +# if b is a number written in binary, and m is either 2 or 3, +# the grammar aims to recognise m:b iif b is a multiple of m +# +# for example, 3:1001 is recognised +# because 9 (0b1001) is a multiple of 3 +# + +from lark import Lark, UnexpectedInput + +parser = Lark.open('multiples.lark', parser='lalr') + +def is_in_grammar(data): + try: + parser.parse(data) + except UnexpectedInput: + return False + return True + +for n_dec in range(100): + n_bin = bin(n_dec)[2:] + assert is_in_grammar('2:{}'.format(n_bin)) == (n_dec % 2 == 0) + assert is_in_grammar('3:{}'.format(n_bin)) == (n_dec % 3 == 0) diff --git a/lark/load_grammar.py b/lark/load_grammar.py index f95d0fa..5e412ec 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -139,7 +139,7 @@ RULES = { 'declare': ['_DECLARE _declare_args _NL'], 'import': ['_IMPORT _import_path _NL', '_IMPORT _import_path _LPAR name_list _RPAR _NL', - '_IMPORT _import_path _TO TERMINAL _NL'], + '_IMPORT _import_path _TO name _NL'], '_import_path': ['import_lib', 'import_rel'], 'import_lib': ['_import_args'], @@ -586,7 +586,7 @@ def import_from_grammar_into_namespace(grammar, namespace, aliases): try: return aliases[name].value except KeyError: - return '%s.%s' % (namespace, name) + return '%s__%s' % (namespace, name) to_import = list(bfs(aliases, rule_dependencies)) for symbol in to_import: @@ -745,7 +745,7 @@ class GrammarLoader: g = import_grammar(grammar_path, base_paths=[base_path]) aliases_dict = dict(zip(names, aliases)) - new_td, new_rd = import_from_grammar_into_namespace(g, '.'.join(dotted_path), aliases_dict) + new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases_dict) term_defs += new_td rule_defs += new_rd diff --git a/tests/grammars/ab.lark b/tests/grammars/ab.lark new file mode 100644 index 0000000..33a985a --- /dev/null +++ b/tests/grammars/ab.lark @@ -0,0 +1,10 @@ +startab: expr + +expr: A B + | A expr B + +A: "a" +B: "b" + +%import common.WS +%ignore WS diff --git a/tests/test_parser.py b/tests/test_parser.py index 03b475d..8e1a234 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -998,11 +998,89 @@ def _make_parser_test(LEXER, PARSER): self.assertEqual(x.children, ['12', 'elephants']) + def test_import_rename(self): + grammar = """ + start: N W + + %import common.NUMBER -> N + %import common.WORD -> W + %import common.WS + %ignore WS + + """ + l = _Lark(grammar) + x = l.parse('12 elephants') + self.assertEqual(x.children, ['12', 'elephants']) + + def test_relative_import(self): l = _Lark_open('test_relative_import.lark', rel_to=__file__) x = l.parse('12 lions') self.assertEqual(x.children, ['12', 'lions']) + + def test_relative_import_rename(self): + l = _Lark_open('test_relative_import_rename.lark', rel_to=__file__) + x = l.parse('12 lions') + self.assertEqual(x.children, ['12', 'lions']) + + + def test_relative_rule_import(self): + l = _Lark_open('test_relative_rule_import.lark', rel_to=__file__) + x = l.parse('xaabby') + self.assertEqual(x.children, [ + 'x', + Tree('expr', ['a', Tree('expr', ['a', 'b']), 'b']), + 'y']) + + + def test_relative_rule_import_drop_ignore(self): + # %ignore rules are dropped on import + l = _Lark_open('test_relative_rule_import_drop_ignore.lark', + rel_to=__file__) + self.assertRaises((ParseError, UnexpectedInput), + l.parse, 'xa abby') + + + def test_relative_rule_import_subrule(self): + l = _Lark_open('test_relative_rule_import_subrule.lark', + rel_to=__file__) + x = l.parse('xaabby') + self.assertEqual(x.children, [ + 'x', + Tree('startab', [ + Tree('grammars__ab__expr', [ + 'a', Tree('grammars__ab__expr', ['a', 'b']), 'b', + ]), + ]), + 'y']) + + + def test_relative_rule_import_subrule_no_conflict(self): + l = _Lark_open( + 'test_relative_rule_import_subrule_no_conflict.lark', + rel_to=__file__) + x = l.parse('xaby') + self.assertEqual(x.children, [Tree('expr', [ + 'x', + Tree('startab', [ + Tree('grammars__ab__expr', ['a', 'b']), + ]), + 'y'])]) + self.assertRaises((ParseError, UnexpectedInput), + l.parse, 'xaxabyby') + + + def test_relative_rule_import_rename(self): + l = _Lark_open('test_relative_rule_import_rename.lark', + rel_to=__file__) + x = l.parse('xaabby') + self.assertEqual(x.children, [ + 'x', + Tree('ab', ['a', Tree('ab', ['a', 'b']), 'b']), + 'y']) + + def test_multi_import(self): grammar = """ start: NUMBER WORD diff --git a/tests/test_relative_import_rename.lark b/tests/test_relative_import_rename.lark new file mode 100644 index 0000000..c411771 --- /dev/null +++ b/tests/test_relative_import_rename.lark @@ -0,0 +1,7 @@ +start: N WORD + +%import .grammars.test.NUMBER -> N +%import common.WORD +%import common.WS +%ignore WS + diff --git a/tests/test_relative_rule_import.lark b/tests/test_relative_rule_import.lark new file mode 100644 index 0000000..e3a33a5 --- /dev/null +++ b/tests/test_relative_rule_import.lark @@ -0,0 +1,7 @@ +start: X expr Y + +X: "x" +Y: "y" + +%import .grammars.ab.expr + diff --git a/tests/test_relative_rule_import_drop_ignore.lark b/tests/test_relative_rule_import_drop_ignore.lark new file mode 100644 index 0000000..e3a33a5 --- /dev/null +++ b/tests/test_relative_rule_import_drop_ignore.lark @@ -0,0 +1,7 @@ +start: X expr Y + +X: "x" +Y: "y" + +%import .grammars.ab.expr + diff --git a/tests/test_relative_rule_import_rename.lark b/tests/test_relative_rule_import_rename.lark new file mode 100644 index 0000000..342b329 --- /dev/null +++ b/tests/test_relative_rule_import_rename.lark @@ -0,0 +1,7 @@ +start: X ab Y + +X: "x" +Y: "y" + +%import .grammars.ab.expr -> ab + diff --git a/tests/test_relative_rule_import_subrule.lark b/tests/test_relative_rule_import_subrule.lark new file mode 100644 index 0000000..94d7f80 --- /dev/null +++ b/tests/test_relative_rule_import_subrule.lark @@ -0,0 +1,7 @@ +start: X startab Y + +X: "x" +Y: "y" + +%import .grammars.ab.startab + diff --git a/tests/test_relative_rule_import_subrule_no_conflict.lark b/tests/test_relative_rule_import_subrule_no_conflict.lark new file mode 100644 index 0000000..839aac1 --- /dev/null +++ b/tests/test_relative_rule_import_subrule_no_conflict.lark @@ -0,0 +1,9 @@ +start: expr + +expr: X startab Y + +X: "x" +Y: "y" + +%import .grammars.ab.startab +