Browse Source

Earley working too

tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.6.0
Erez Shinan 6 years ago
parent
commit
cf7ddeee88
5 changed files with 16 additions and 13 deletions
  1. +1
    -1
      examples/python_parser.py
  2. +1
    -1
      lark/parse_tree_builder.py
  3. +4
    -3
      lark/parser_frontends.py
  4. +5
    -4
      lark/parsers/earley.py
  5. +5
    -4
      lark/parsers/xearley.py

+ 1
- 1
examples/python_parser.py View File

@@ -78,6 +78,6 @@ def test_earley_equals_lalr():

if __name__ == '__main__':
test_python_lib()
# test_earley_equals_lalr()
test_earley_equals_lalr()
# python_parser3.parse(_read(sys.argv[1]) + '\n')


+ 1
- 1
lark/parse_tree_builder.py View File

@@ -1,4 +1,4 @@
from .common import is_terminal, GrammarError
from .common import GrammarError
from .utils import suppress
from .lexer import Token
from .grammar import Rule


+ 4
- 3
lark/parser_frontends.py View File

@@ -4,9 +4,10 @@ from .utils import get_regexp_width
from .parsers.grammar_analysis import GrammarAnalyzer
from .lexer import Lexer, ContextualLexer, Token

from .common import is_terminal, GrammarError
from .common import GrammarError
from .parsers import lalr_parser, earley, xearley, resolve_ambig, cyk
from .tree import Tree
from .grammar import Terminal

class WithLexer:
def init_traditional_lexer(self, lexer_conf):
@@ -96,7 +97,7 @@ class Earley(WithLexer):
resolve_ambiguity=get_ambiguity_resolver(options))

def match(self, term, token):
return term == token.type
return term.name == token.type

def parse(self, text):
tokens = self.lex(text)
@@ -117,7 +118,7 @@ class XEarley:
)

def match(self, term, text, index=0):
return self.regexps[term].match(text, index)
return self.regexps[term.name].match(text, index)

def _prepare_match(self, lexer_conf):
self.regexps = {}


+ 5
- 4
lark/parsers/earley.py View File

@@ -13,9 +13,10 @@
# Author: Erez Shinan (2017)
# Email : erezshin@gmail.com

from ..common import ParseError, UnexpectedToken, is_terminal
from ..common import ParseError, UnexpectedToken
from ..tree import Tree, Transformer_NoRecurse
from .grammar_analysis import GrammarAnalyzer
from ..grammar import NonTerminal


class Derivation(Tree):
@@ -127,7 +128,7 @@ class Column:
self.completed[item_key] = item
self.to_reduce.append(item)
else:
if is_terminal(item.expect):
if item.expect.is_term:
self.to_scan.append(item)
else:
k = item_key if self.predict_all else item
@@ -161,13 +162,13 @@ class Parser:

def parse(self, stream, start_symbol=None):
# Define parser functions
start_symbol = start_symbol or self.parser_conf.start
start_symbol = NonTerminal(start_symbol or self.parser_conf.start)

_Item = Item
match = self.term_matcher

def predict(nonterm, column):
assert not is_terminal(nonterm), nonterm
assert not nonterm.is_term, nonterm
return [_Item(rule, 0, column, None) for rule in self.predictions[nonterm]]

def complete(item):


+ 5
- 4
lark/parsers/xearley.py View File

@@ -20,10 +20,11 @@

from collections import defaultdict

from ..common import ParseError, is_terminal
from ..common import ParseError
from ..lexer import Token, UnexpectedInput
from ..tree import Tree
from .grammar_analysis import GrammarAnalyzer
from ..grammar import NonTerminal, Terminal

from .earley import ApplyCallbacks, Item, Column

@@ -32,7 +33,7 @@ class Parser:
self.analysis = GrammarAnalyzer(parser_conf)
self.parser_conf = parser_conf
self.resolve_ambiguity = resolve_ambiguity
self.ignore = list(ignore)
self.ignore = [Terminal(t) for t in ignore]
self.predict_all = predict_all

self.FIRST = self.analysis.FIRST
@@ -47,7 +48,7 @@ class Parser:

def parse(self, stream, start_symbol=None):
# Define parser functions
start_symbol = start_symbol or self.parser_conf.start
start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
delayed_matches = defaultdict(list)
match = self.term_matcher

@@ -55,7 +56,7 @@ class Parser:
text_column = 0

def predict(nonterm, column):
assert not is_terminal(nonterm), nonterm
assert not nonterm.is_term, nonterm
return [Item(rule, 0, column, None) for rule in self.predictions[nonterm]]

def complete(item):


Loading…
Cancel
Save