Earley working too

examples/python_parser.py

@@ -78,6 +78,6 @@ def test_earley_equals_lalr():
 
 if __name__ == '__main__':
     test_python_lib()
-    # test_earley_equals_lalr()
+    test_earley_equals_lalr()
     # python_parser3.parse(_read(sys.argv[1]) + '\n')
 

lark/parse_tree_builder.py

@@ -1,4 +1,4 @@
-from .common import is_terminal, GrammarError
+from .common import GrammarError
 from .utils import suppress
 from .lexer import Token
 from .grammar import Rule

lark/parser_frontends.py

@@ -4,9 +4,10 @@ from .utils import get_regexp_width
 from .parsers.grammar_analysis import GrammarAnalyzer
 from .lexer import Lexer, ContextualLexer, Token
 
-from .common import is_terminal, GrammarError
+from .common import GrammarError
 from .parsers import lalr_parser, earley, xearley, resolve_ambig, cyk
 from .tree import Tree
+from .grammar import Terminal
 
 class WithLexer:
     def init_traditional_lexer(self, lexer_conf):
@@ -96,7 +97,7 @@ class Earley(WithLexer):
                                     resolve_ambiguity=get_ambiguity_resolver(options))
 
     def match(self, term, token):
-        return term == token.type
+        return term.name == token.type
 
     def parse(self, text):
         tokens = self.lex(text)
@@ -117,7 +118,7 @@ class XEarley:
                                     )
 
     def match(self, term, text, index=0):
-        return self.regexps[term].match(text, index)
+        return self.regexps[term.name].match(text, index)
 
     def _prepare_match(self, lexer_conf):
         self.regexps = {}

lark/parsers/earley.py

@@ -13,9 +13,10 @@
 # Author: Erez Shinan (2017)
 # Email : erezshin@gmail.com
 
-from ..common import ParseError, UnexpectedToken, is_terminal
+from ..common import ParseError, UnexpectedToken
 from ..tree import Tree, Transformer_NoRecurse
 from .grammar_analysis import GrammarAnalyzer
+from ..grammar import NonTerminal
 
 
 class Derivation(Tree):
@@ -127,7 +128,7 @@ class Column:
                     self.completed[item_key] = item
                 self.to_reduce.append(item)
             else:
-                if is_terminal(item.expect):
+                if item.expect.is_term:
                     self.to_scan.append(item)
                 else:
                     k = item_key if self.predict_all else item
@@ -161,13 +162,13 @@ class Parser:
 
     def parse(self, stream, start_symbol=None):
         # Define parser functions
-        start_symbol = start_symbol or self.parser_conf.start
+        start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
 
         _Item = Item
         match = self.term_matcher
 
         def predict(nonterm, column):
-            assert not is_terminal(nonterm), nonterm
+            assert not nonterm.is_term, nonterm
             return [_Item(rule, 0, column, None) for rule in self.predictions[nonterm]]
 
         def complete(item):

lark/parsers/xearley.py

@@ -20,10 +20,11 @@
 
 from collections import defaultdict
 
-from ..common import ParseError, is_terminal
+from ..common import ParseError
 from ..lexer import Token, UnexpectedInput
 from ..tree import Tree
 from .grammar_analysis import GrammarAnalyzer
+from ..grammar import NonTerminal, Terminal
 
 from .earley import ApplyCallbacks, Item, Column
 
@@ -32,7 +33,7 @@ class Parser:
         self.analysis = GrammarAnalyzer(parser_conf)
         self.parser_conf = parser_conf
         self.resolve_ambiguity = resolve_ambiguity
-        self.ignore = list(ignore)
+        self.ignore = [Terminal(t) for t in ignore]
         self.predict_all = predict_all
 
         self.FIRST = self.analysis.FIRST
@@ -47,7 +48,7 @@ class Parser:
 
     def parse(self, stream, start_symbol=None):
         # Define parser functions
-        start_symbol = start_symbol or self.parser_conf.start
+        start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
         delayed_matches = defaultdict(list)
         match = self.term_matcher
 
@@ -55,7 +56,7 @@ class Parser:
         text_column = 0
 
         def predict(nonterm, column):
-            assert not is_terminal(nonterm), nonterm
+            assert not nonterm.is_term, nonterm
             return [Item(rule, 0, column, None) for rule in self.predictions[nonterm]]
 
         def complete(item):