diff --git a/lark/load_grammar.py b/lark/load_grammar.py index dbf4a1f..54c7ca0 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -231,12 +231,11 @@ class EBNF_to_BNF(Transformer_InPlace): return not sym.name.startswith('_') if isinstance(sym, Terminal): return keep_all_tokens or not sym.filter_out - assert False + if isinstance(sym, Symbol): + return False + assert False, sym - if any(rule.scan_values(will_not_get_removed)): - empty = _EMPTY - else: - empty = ST('expansion', []) + empty = ST('expansion', [_EMPTY] * len(list(rule.scan_values(will_not_get_removed)))) return ST('expansions', [rule, empty]) diff --git a/tests/test_parser.py b/tests/test_parser.py index ff4e064..58b1ef5 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2293,6 +2293,15 @@ def _make_parser_test(LEXER, PARSER): self.assertEqual(p.parse("abba").children, ['a', None, 'b', 'b', 'a', None]) self.assertEqual(p.parse("cbbbb").children, [None, 'c', 'b', 'b', 'b', 'b', None, None]) + p = _Lark("""!start: ["a" "b" "c"] """, maybe_placeholders=True) + self.assertEqual(p.parse("").children, [None, None, None]) + self.assertEqual(p.parse("abc").children, ['a', 'b', 'c']) + + p = _Lark("""!start: ["a" ["b" "c"]] """, maybe_placeholders=True) + self.assertEqual(p.parse("").children, [None, None, None]) + self.assertEqual(p.parse("a").children, ['a', None, None]) + self.assertEqual(p.parse("abc").children, ['a', 'b', 'c']) + def test_escaped_string(self): "Tests common.ESCAPED_STRING"