diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index f9dfbfe..8b8b411 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -27,6 +27,44 @@ class Item: def __hash__(self): return hash((self.rule, self.ptr, self.start)) + def __repr__(self): + before = map(str, self.rule.expansion[:self.ptr]) + after = map(str, self.rule.expansion[self.ptr:]) + return '<(%d) %s : %s * %s>' % (self.start, self.rule.origin, ' '.join(before), ' '.join(after)) + + +class NewsList(list): + def __init__(self, initial=None): + list.__init__(self, initial or []) + self.last_iter = 0 + + def get_news(self): + i = self.last_iter + self.last_iter = len(self) + return self[i:] + + +class Column: + def __init__(self): + self.to_reduce = NewsList() + self.to_predict = NewsList() + self.to_scan = NewsList() + self.item_count = 0 + + def add(self, items): + self.item_count += len(items) + for item in items: + if item.is_complete: + if item not in self.to_reduce: # Avoid infinite loop + self.to_reduce.append(item) + else: + if is_terminal(item.expect): + self.to_scan.append(item) + else: + self.to_predict.append(item) + + def __nonzero__(self): + return bool(self.item_count) class Parser: def __init__(self, parser_conf): @@ -39,53 +77,53 @@ class Parser: if rule.origin != '$root': # XXX kinda ugly a = rule.alias self.postprocess[rule] = a if callable(a) else getattr(parser_conf.callback, a) - self.predictions[rule.origin] = [(x.rule, x.index) for x in self.analysis.expand_rule(rule.origin)] + self.predictions[rule.origin] = [x.rule for x in self.analysis.expand_rule(rule.origin)] def parse(self, stream): # Define parser functions - def predict(symbol, i): - assert not is_terminal(symbol), symbol - return {Item(rule, index, i, []) for rule, index in self.predictions[symbol]} + def predict(nonterm, i): + assert not is_terminal(nonterm), nonterm + return [Item(rule, 0, i, []) for rule in self.predictions[nonterm]] def complete(item, table): + name = item.rule.origin item.data = self.postprocess[item.rule](item.data) - return {old_item.advance(item.data) for old_item in table[item.start] - if not old_item.is_complete and old_item.expect == item.rule.origin} + return [i.advance(item.data) for i in table[item.start].to_predict + if i.expect == name] def process_column(i, token): assert i == len(table)-1 cur_set = table[i] - next_set = set() - - to_process = cur_set - while to_process: - new_items = set() - for item in to_process: - if item.is_complete: - new_items |= complete(item, table) - else: - if is_terminal(item.expect): - # scan - match = item.expect[0](token) if callable(item.expect[0]) else item.expect[0] == token.type - if match: - next_set.add(item.advance(stream[i])) - else: - if item.ptr: # part of an already predicted batch - new_items |= predict(item.expect, i) - - to_process = new_items - cur_set # TODO: is this precaution necessary? - cur_set |= to_process + next_set = Column() + while True: + to_predict = {x.expect for x in cur_set.to_predict.get_news() + if x.ptr} # if not part of an already predicted batch + to_reduce = cur_set.to_reduce.get_news() + if not (to_predict or to_reduce): + break + + for nonterm in to_predict: + cur_set.add( predict(nonterm, i) ) + for item in to_reduce: + cur_set.add( complete(item, table) ) + + + for item in cur_set.to_scan.get_news(): + match = item.expect[0](token) if callable(item.expect[0]) else item.expect[0] == token.type + if match: + next_set.add([item.advance(stream[i])]) if not next_set and token.type != '$end': - expect = filter(is_terminal, [x.expect for x in cur_set if not x.is_complete]) + expect = [i.expect for i in cur_set.to_scan] raise UnexpectedToken(token, expect, stream, i) table.append(next_set) # Main loop starts - table = [predict(self.start, 0)] + table = [Column()] + table[0].add(predict(self.start, 0)) for i, char in enumerate(stream): process_column(i, char) @@ -93,8 +131,8 @@ class Parser: process_column(len(stream), EndToken()) # Parse ended. Now build a parse tree - solutions = [n.data for n in table[len(stream)] - if n.is_complete and n.rule.origin==self.start and n.start==0] + solutions = [n.data for n in table[len(stream)].to_reduce + if n.rule.origin==self.start and n.start==0] if not solutions: raise ParseError('Incomplete parse: Could not find a solution to input') diff --git a/lark/parsers/grammar_analysis.py b/lark/parsers/grammar_analysis.py index c03d9ae..3428713 100644 --- a/lark/parsers/grammar_analysis.py +++ b/lark/parsers/grammar_analysis.py @@ -13,7 +13,7 @@ class Rule(object): self.alias = alias def __repr__(self): - return '<%s : %s>' % (self.origin, ' '.join(self.expansion)) + return '<%s : %s>' % (self.origin, ' '.join(map(unicode,self.expansion))) class RulePtr(object): def __init__(self, rule, index):