diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 94bb97f..281b289 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -345,12 +345,11 @@ class EBNF_to_BNF(Transformer_InPlace): return not sym.name.startswith('_') if isinstance(sym, Terminal): return keep_all_tokens or not sym.filter_out - assert False + if sym is _EMPTY: + return False + assert False, sym - if any(rule.scan_values(will_not_get_removed)): - empty = _EMPTY - else: - empty = ST('expansion', []) + empty = ST('expansion', [_EMPTY] * len(list(rule.scan_values(will_not_get_removed)))) return ST('expansions', [rule, empty]) diff --git a/tests/test_parser.py b/tests/test_parser.py index dab69f7..6974a7c 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2292,6 +2292,15 @@ def _make_parser_test(LEXER, PARSER): self.assertEqual(p.parse("abba").children, ['a', None, 'b', 'b', 'a', None]) self.assertEqual(p.parse("cbbbb").children, [None, 'c', 'b', 'b', 'b', 'b', None, None]) + p = _Lark("""!start: ["a" "b" "c"] """, maybe_placeholders=True) + self.assertEqual(p.parse("").children, [None, None, None]) + self.assertEqual(p.parse("abc").children, ['a', 'b', 'c']) + + p = _Lark("""!start: ["a" ["b" "c"]] """, maybe_placeholders=True) + self.assertEqual(p.parse("").children, [None, None, None]) + self.assertEqual(p.parse("a").children, ['a', None, None]) + self.assertEqual(p.parse("abc").children, ['a', 'b', 'c']) + def test_escaped_string(self): "Tests common.ESCAPED_STRING"