Fixed a deep bug in grammar analysis involving empty rules (Issue #250)

lark/parsers/cyk.py

@@ -199,7 +199,8 @@ class CnfWrapper(object):
         for r in self.rules:
             # Validate that the grammar is CNF and populate auxiliary data structures.
             assert isinstance(r.lhs, NT), r
-            assert len(r.rhs) in [1, 2], r
+            if len(r.rhs) not in [1, 2]:
+                raise ParseError("CYK doesn't support empty rules")
             if len(r.rhs) == 1 and isinstance(r.rhs[0], T):
                 self.terminal_rules[r.rhs[0]].append(r)
             elif len(r.rhs) == 2 and all(isinstance(x, NT) for x in r.rhs):

lark/parsers/grammar_analysis.py

@@ -92,7 +92,7 @@ def calculate_sets(rules):
 
         for rule in rules:
             for i, sym in enumerate(rule.expansion):
-                if i==len(rule.expansion)-1 or set(rule.expansion[i:]) <= NULLABLE:
+                if i==len(rule.expansion)-1 or set(rule.expansion[i+1:]) <= NULLABLE:
                     if update_set(FOLLOW[sym], FOLLOW[rule.origin]):
                         changed = True
 

tests/test_parser.py

@@ -1237,6 +1237,17 @@ def _make_parser_test(LEXER, PARSER):
                 self.assertEqual(tok.end_line, 2)
                 self.assertEqual(tok.end_column, 6)
 
+        @unittest.skipIf(PARSER=='cyk', "Empty rules")
+        def test_empty_end(self):
+            p = _Lark("""
+                start: b c d
+                b: "B"
+                c: | "C"
+                d: | "D"
+            """)
+            res = p.parse('B')
+            self.assertEqual(len(res.children), 3)
+
 
 
     _NAME = "Test" + PARSER.capitalize() + LEXER.capitalize()