From c3fc2266e199e72f8b0ba892d87fd66ea7f93fc9 Mon Sep 17 00:00:00 2001 From: ornariece <25489980+ornariece@users.noreply.github.com> Date: Sat, 26 Jun 2021 16:04:57 +0200 Subject: [PATCH 1/2] Fix #696 now providing the correct amount of placeholders --- lark/load_grammar.py | 9 ++++----- tests/test_parser.py | 9 +++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lark/load_grammar.py b/lark/load_grammar.py index dbf4a1f..54c7ca0 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -231,12 +231,11 @@ class EBNF_to_BNF(Transformer_InPlace): return not sym.name.startswith('_') if isinstance(sym, Terminal): return keep_all_tokens or not sym.filter_out - assert False + if isinstance(sym, Symbol): + return False + assert False, sym - if any(rule.scan_values(will_not_get_removed)): - empty = _EMPTY - else: - empty = ST('expansion', []) + empty = ST('expansion', [_EMPTY] * len(list(rule.scan_values(will_not_get_removed)))) return ST('expansions', [rule, empty]) diff --git a/tests/test_parser.py b/tests/test_parser.py index ff4e064..58b1ef5 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2293,6 +2293,15 @@ def _make_parser_test(LEXER, PARSER): self.assertEqual(p.parse("abba").children, ['a', None, 'b', 'b', 'a', None]) self.assertEqual(p.parse("cbbbb").children, [None, 'c', 'b', 'b', 'b', 'b', None, None]) + p = _Lark("""!start: ["a" "b" "c"] """, maybe_placeholders=True) + self.assertEqual(p.parse("").children, [None, None, None]) + self.assertEqual(p.parse("abc").children, ['a', 'b', 'c']) + + p = _Lark("""!start: ["a" ["b" "c"]] """, maybe_placeholders=True) + self.assertEqual(p.parse("").children, [None, None, None]) + self.assertEqual(p.parse("a").children, ['a', None, None]) + self.assertEqual(p.parse("abc").children, ['a', 'b', 'c']) + def test_escaped_string(self): "Tests common.ESCAPED_STRING" From c24d6ecff51ed5800ad3d44a69a53841ca8dd4ec Mon Sep 17 00:00:00 2001 From: ornariece <25489980+ornariece@users.noreply.github.com> Date: Sun, 27 Jun 2021 12:33:05 +0200 Subject: [PATCH 2/2] Fix #696 testing for is _EMPTY --- lark/load_grammar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 54c7ca0..7c3d672 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -231,7 +231,7 @@ class EBNF_to_BNF(Transformer_InPlace): return not sym.name.startswith('_') if isinstance(sym, Terminal): return keep_all_tokens or not sym.filter_out - if isinstance(sym, Symbol): + if sym is _EMPTY: return False assert False, sym