From 8415fa26a3d3d2b81e64e4fe440faab15b53db49 Mon Sep 17 00:00:00 2001
From: night199uk <night199uk@hermitcrabslab.com>
Date: Tue, 4 Dec 2018 21:34:20 +0100
Subject: [PATCH 1/4] Add a pydot visualizer for the SPPF.

---
 lark/parsers/earley_forest.py | 95 +++++++++++++++++++++++++++++++++--
 1 file changed, 92 insertions(+), 3 deletions(-)

diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py
index e4e3e36..e5038d9 100644
--- a/lark/parsers/earley_forest.py
+++ b/lark/parsers/earley_forest.py
@@ -7,6 +7,7 @@ Full reference and more details is here:
 http://www.bramvandersanden.com/post/2014/06/shared-packed-parse-forest/
 """
 
+from random import randint
 from ..tree import Tree
 from ..exceptions import ParseError
 from ..lexer import Token
@@ -15,6 +16,7 @@ from ..grammar import NonTerminal, Terminal
 from .earley_common import Column, Derivation
 
 from collections import deque
+from importlib import import_module
 
 class ForestNode(object):
     pass
@@ -61,7 +63,13 @@ class SymbolNode(ForestNode):
         return hash((self.s, self.start.i, self.end.i))
 
     def __repr__(self):
-        symbol = self.s.name if isinstance(self.s, (NonTerminal, Terminal)) else self.s[0].origin.name
+        if self.is_intermediate:
+            rule = self.s[0]
+            ptr = self.s[1]
+            names = [ "{}*".format(expansion.name) if index == ptr else expansion.name for index, expansion in enumerate(rule.expansion) ]
+            symbol = "{} ::= {}".format(rule.origin.name, ' '.join(names))
+        else:
+            symbol = self.s.name
         return "(%s, %d, %d, %d)" % (symbol, self.start.i, self.end.i, self.priority if self.priority is not None else 0)
 
 class PackedNode(ForestNode):
@@ -105,8 +113,14 @@ class PackedNode(ForestNode):
         return self._hash
 
     def __repr__(self):
-        symbol = self.s.name if isinstance(self.s, (NonTerminal, Terminal)) else self.s[0].origin.name
-        return "{%s, %d, %s, %s, %s}" % (symbol, self.start.i, self.left, self.right, self.priority if self.priority is not None else 0)
+        if isinstance(self.s, tuple):
+            rule = self.s[0]
+            ptr = self.s[1]
+            names = [ "{}*".format(expansion.name) if index == ptr else expansion.name for index, expansion in enumerate(rule.expansion) ]
+            symbol = "{} ::= {}".format(rule.origin.name, ' '.join(names))
+        else:
+            symbol = self.s.name
+        return "{%s, %d, %d}" % (symbol, self.start.i, self.priority if self.priority is not None else 0)
 
 class ForestVisitor(object):
     """
@@ -351,3 +365,78 @@ class ForestToAmbiguousTreeVisitor(ForestVisitor):
                 self.output_stack[-1].children.append(result)
             else:
                 self.result = result
+
+class ForestToPyDotVisitor(ForestVisitor):
+    """
+    A Forest visitor which writes the SPPF to a PNG.
+
+    The SPPF can get really large, really quickly because
+    of the amount of meta-data it stores, so this is probably
+    only useful for trivial trees and learning how the SPPF
+    is structured.
+    """
+    def __init__(self, rankdir="TB"):
+        self.pydot = import_module('pydot')
+        self.graph = self.pydot.Dot(graph_type='digraph', rankdir=rankdir)
+
+    def go(self, root, filename):
+        super(ForestToPyDotVisitor, self).go(root)
+        self.graph.write_png(filename)
+
+    def visit_token_node(self, node):
+        graph_node_id = str(id(node))
+        graph_node_label = "\"{}\"".format(node.value.replace('"', '\\"'))
+        graph_node_color = 0x808080
+        graph_node_style = "filled"
+        graph_node_shape = "polygon"
+        graph_node = self.pydot.Node(graph_node_id, style=graph_node_style, fillcolor="#{:06x}".format(graph_node_color), shape=graph_node_shape, label=graph_node_label)
+        self.graph.add_node(graph_node)
+
+    def visit_packed_node_in(self, node):
+        graph_node_id = str(id(node))
+        graph_node_label = repr(node)
+        graph_node_color = 0x808080
+        graph_node_style = "filled"
+        graph_node_shape = "diamond"
+        graph_node = self.pydot.Node(graph_node_id, style=graph_node_style, fillcolor="#{:06x}".format(graph_node_color), shape=graph_node_shape, label=graph_node_label)
+        self.graph.add_node(graph_node)
+        return iter([node.left, node.right])
+
+    def visit_packed_node_out(self, node):
+        graph_node_id = str(id(node))
+        graph_node = self.graph.get_node(graph_node_id)[0]
+        for child in [node.left, node.right]:
+            if child is not None:
+                child_graph_node_id = str(id(child))
+                child_graph_node = self.graph.get_node(child_graph_node_id)[0]
+                self.graph.add_edge(self.pydot.Edge(graph_node, child_graph_node))
+            else:
+                #### Try and be above the Python object ID range; probably impl. specific, but maybe this is okay.
+                child_graph_node_id = str(randint(100000000000000000000000000000,123456789012345678901234567890))
+                child_graph_node_style = "invis"
+                child_graph_node = self.pydot.Node(child_graph_node_id, style=child_graph_node_style, label="None")
+                child_edge_style = "invis"
+                self.graph.add_node(child_graph_node)
+                self.graph.add_edge(self.pydot.Edge(graph_node, child_graph_node, style=child_edge_style))
+
+    def visit_symbol_node_in(self, node):
+        graph_node_id = str(id(node))
+        graph_node_label = repr(node)
+        graph_node_color = 0x808080
+        graph_node_style = "filled"
+        if node.is_intermediate:
+            graph_node_shape = "ellipse"
+        else:
+            graph_node_shape = "rectangle"
+        graph_node = self.pydot.Node(graph_node_id, style=graph_node_style, fillcolor="#{:06x}".format(graph_node_color), shape=graph_node_shape, label=graph_node_label)
+        self.graph.add_node(graph_node)
+        return iter(node.children)
+
+    def visit_symbol_node_out(self, node):
+        graph_node_id = str(id(node))
+        graph_node = self.graph.get_node(graph_node_id)[0]
+        for child in node.children:
+            child_graph_node_id = str(id(child))
+            child_graph_node = self.graph.get_node(child_graph_node_id)[0]
+            self.graph.add_edge(self.pydot.Edge(graph_node, child_graph_node))
+

From 8fa8ac36fc23c6a34f9af362405ca854b8e03e43 Mon Sep 17 00:00:00 2001
From: night199uk <night199uk@hermitcrabslab.com>
Date: Wed, 12 Dec 2018 14:45:52 +0100
Subject: [PATCH 2/4] Remove Earley Column

We can replace Earley Columns with basic python sets
for improved performance and simplicity.
---
 lark/parsers/earley.py        | 61 ++++++++++++++++++--------------
 lark/parsers/earley_common.py | 26 +++-----------
 lark/parsers/earley_forest.py | 10 +++---
 lark/parsers/xearley.py       | 66 +++++++++++++++++++----------------
 4 files changed, 80 insertions(+), 83 deletions(-)

diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py
index 02bc5d4..7600915 100644
--- a/lark/parsers/earley.py
+++ b/lark/parsers/earley.py
@@ -16,7 +16,7 @@ from ..visitors import Transformer_InPlace, v_args
 from ..exceptions import ParseError, UnexpectedToken
 from .grammar_analysis import GrammarAnalyzer
 from ..grammar import NonTerminal
-from .earley_common import Column, Item
+from .earley_common import Item
 from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode
 
 from collections import deque, defaultdict
@@ -48,19 +48,24 @@ class Parser:
         # Define parser functions
         start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
         match = self.term_matcher
-        held_completions = defaultdict(list)
+
+        # Held Completions (H in E.Scotts paper).
+        held_completions = {}
+
+        # Cache for nodes & tokens created in a particular parse step.
         node_cache = {}
         token_cache = {}
+        columns = []
 
         def make_symbol_node(s, start, end):
-            label = (s, start.i, end.i)
+            label = (s, start, end)
             if label in node_cache:
                 node = node_cache[label]
             else:
                 node = node_cache[label] = SymbolNode(s, start, end)
             return node
 
-        def predict_and_complete(column, to_scan):
+        def predict_and_complete(i, to_scan):
             """The core Earley Predictor and Completer.
 
             At each stage of the input, we handling any completed items (things
@@ -70,15 +75,16 @@ class Parser:
             which can be added to the scan list for the next scanner cycle."""
             held_completions.clear()
 
+            column = columns[i]
             # R (items) = Ei (column.items)
-            items = deque(column.items)
+            items = deque(column)
             while items:
                 item = items.pop()    # remove an element, A say, from R
 
                 ### The Earley completer
                 if item.is_complete:   ### (item.s == string)
                     if item.node is None:
-                        item.node = make_symbol_node(item.s, item.start, column)
+                        item.node = make_symbol_node(item.s, item.start, i)
                         item.node.add_family(item.s, item.rule, item.start, None, None)
 
                     # Empty has 0 length. If we complete an empty symbol in a particular
@@ -86,19 +92,19 @@ class Parser:
                     # any predictions that result, that themselves require empty. Avoids
                     # infinite recursion on empty symbols.
                     # held_completions is 'H' in E.Scott's paper.
-                    is_empty_item = item.start.i == column.i
+                    is_empty_item = item.start == i
                     if is_empty_item:
                         held_completions[item.rule.origin] = item.node
 
-                    originators = [originator for originator in item.start.items if originator.expect is not None and originator.expect == item.s]
+                    originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s]
                     for originator in originators:
                         new_item = originator.advance()
-                        new_item.node = make_symbol_node(new_item.s, originator.start, column)
+                        new_item.node = make_symbol_node(new_item.s, originator.start, i)
                         new_item.node.add_family(new_item.s, new_item.rule, new_item.start, originator.node, item.node)
                         if new_item.expect in self.TERMINALS:
                             # Add (B :: aC.B, h, y) to Q
                             to_scan.add(new_item)
-                        elif new_item not in column.items:
+                        elif new_item not in column:
                             # Add (B :: aC.B, h, y) to Ei and R
                             column.add(new_item)
                             items.append(new_item)
@@ -107,24 +113,24 @@ class Parser:
                 elif item.expect in self.NON_TERMINALS: ### (item.s == lr0)
                     new_items = []
                     for rule in self.predictions[item.expect]:
-                        new_item = Item(rule, 0, column)
+                        new_item = Item(rule, 0, i)
                         new_items.append(new_item)
 
                     # Process any held completions (H).
                     if item.expect in held_completions:
                         new_item = item.advance()
-                        new_item.node = make_symbol_node(new_item.s, item.start, column)
+                        new_item.node = make_symbol_node(new_item.s, item.start, i)
                         new_item.node.add_family(new_item.s, new_item.rule, new_item.start, item.node, held_completions[item.expect])
                         new_items.append(new_item)
 
                     for new_item in new_items:
                         if new_item.expect in self.TERMINALS:
                             to_scan.add(new_item)
-                        elif new_item not in column.items:
+                        elif new_item not in column:
                             column.add(new_item)
                             items.append(new_item)
 
-        def scan(i, token, column, to_scan):
+        def scan(i, token, to_scan):
             """The core Earley Scanner.
 
             This is a custom implementation of the scanner that uses the
@@ -132,12 +138,14 @@ class Parser:
             Earley predictor, based on the previously completed tokens.
             This ensures that at each phase of the parse we have a custom
             lexer context, allowing for more complex ambiguities."""
-            next_set = Column(i+1, self.FIRST)
             next_to_scan = set()
+            next_set = set()
+            columns.append(next_set)
+
             for item in set(to_scan):
                 if match(item.expect, token):
                     new_item = item.advance()
-                    new_item.node = make_symbol_node(new_item.s, new_item.start, column)
+                    new_item.node = make_symbol_node(new_item.s, new_item.start, i)
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token)
 
                     if new_item.expect in self.TERMINALS:
@@ -151,11 +159,10 @@ class Parser:
                 expect = {i.expect.name for i in to_scan}
                 raise UnexpectedToken(token, expect, considered_rules = set(to_scan))
 
-            return next_set, next_to_scan
+            return next_to_scan
 
         # Main loop starts
-        column0 = Column(0, self.FIRST)
-        column = column0
+        columns.append(set())
 
         ## The scan buffer. 'Q' in E.Scott's paper.
         to_scan = set()
@@ -164,32 +171,34 @@ class Parser:
         # Add predicted items to the first Earley set (for the predictor) if they
         # result in a non-terminal, or the scanner if they result in a terminal.
         for rule in self.predictions[start_symbol]:
-            item = Item(rule, 0, column0)
+            item = Item(rule, 0, 0)
             if item.expect in self.TERMINALS:
                 to_scan.add(item)
             else:
-                column.add(item)
+                columns[0].add(item)
 
         ## The main Earley loop.
         # Run the Prediction/Completion cycle for any Items in the current Earley set.
         # Completions will be added to the SPPF tree, and predictions will be recursively
         # processed down to terminals/empty nodes to be added to the scanner for the next
         # step.
-        for i, token in enumerate(stream):
-            predict_and_complete(column, to_scan)
+        i = 0
+        for token in stream:
+            predict_and_complete(i, to_scan)
 
             # Clear the node_cache and token_cache, which are only relevant for each
             # step in the Earley pass.
             node_cache.clear()
             token_cache.clear()
-            column, to_scan = scan(i, token, column, to_scan)
+            to_scan = scan(i, token, to_scan)
+            i += 1
 
-        predict_and_complete(column, to_scan)
+        predict_and_complete(i, to_scan)
 
         ## Column is now the final column in the parse. If the parse was successful, the start
         # symbol should have been completed in the last step of the Earley cycle, and will be in
         # this column. Find the item for the start_symbol, which is the root of the SPPF tree.
-        solutions = [n.node for n in column.items if n.is_complete and n.node is not None and n.s == start_symbol and n.start is column0]
+        solutions = [n.node for n in columns[i] if n.is_complete and n.node is not None and n.s == start_symbol and n.start == 0]
 
         if not solutions:
             raise ParseError('Incomplete parse: Could not find a solution to input')
diff --git a/lark/parsers/earley_common.py b/lark/parsers/earley_common.py
index d17abe4..74dd388 100644
--- a/lark/parsers/earley_common.py
+++ b/lark/parsers/earley_common.py
@@ -35,6 +35,7 @@ class Item(object):
 
     __slots__ = ('s', 'rule', 'ptr', 'start', 'is_complete', 'expect', 'node', '_hash')
     def __init__(self, rule, ptr, start):
+        assert isinstance(start, int), "start is not an int"
         self.is_complete = len(rule.expansion) == ptr
         self.rule = rule    # rule
         self.ptr = ptr      # ptr
@@ -46,35 +47,16 @@ class Item(object):
         else:
             self.s = (rule, ptr)
             self.expect = rule.expansion[ptr]
-        self._hash = hash((self.s, self.start.i))
+        self._hash = hash((self.s, self.start))
 
     def advance(self):
         return self.__class__(self.rule, self.ptr + 1, self.start)
 
     def __eq__(self, other):
-        return self is other or (self.s == other.s and self.start.i == other.start.i)
+        return self is other or (self.s == other.s and self.start == other.start)
 
     def __hash__(self):
         return self._hash
 
     def __repr__(self):
-        return '%s (%d)' % (self.s if self.is_complete else self.rule.origin, self.start.i)
-
-class Column:
-    "An entry in the table, aka Earley Chart. Contains lists of items."
-    def __init__(self, i, FIRST):
-        self.i = i
-        self.items = set()
-        self.FIRST = FIRST
-
-    def add(self, item):
-        """Sort items into scan/predict/reduce newslists
-
-        Makes sure only unique items are added.
-        """
-        self.items.add(item)
-
-    def __bool__(self):
-        return bool(self.items)
-
-    __nonzero__ = __bool__  # Py2 backwards-compatibility
+        return '%s (%d)' % (self.s if self.is_complete else self.rule.origin, self.start)
diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py
index e5038d9..730ebe1 100644
--- a/lark/parsers/earley_forest.py
+++ b/lark/parsers/earley_forest.py
@@ -13,7 +13,7 @@ from ..exceptions import ParseError
 from ..lexer import Token
 from ..utils import Str
 from ..grammar import NonTerminal, Terminal
-from .earley_common import Column, Derivation
+from .earley_common import Derivation
 
 from collections import deque
 from importlib import import_module
@@ -60,7 +60,7 @@ class SymbolNode(ForestNode):
         return self is other or (self.s == other.s and self.start == other.start and self.end is other.end)
 
     def __hash__(self):
-        return hash((self.s, self.start.i, self.end.i))
+        return hash((self.s, self.start, self.end))
 
     def __repr__(self):
         if self.is_intermediate:
@@ -70,7 +70,7 @@ class SymbolNode(ForestNode):
             symbol = "{} ::= {}".format(rule.origin.name, ' '.join(names))
         else:
             symbol = self.s.name
-        return "(%s, %d, %d, %d)" % (symbol, self.start.i, self.end.i, self.priority if self.priority is not None else 0)
+        return "(%s, %d, %d, %d)" % (symbol, self.start, self.end, self.priority if self.priority is not None else 0)
 
 class PackedNode(ForestNode):
     """
@@ -85,7 +85,7 @@ class PackedNode(ForestNode):
         self.left = left
         self.right = right
         self.priority = None
-        self._hash = hash((self.s, self.start.i, self.left, self.right))
+        self._hash = hash((self.s, self.start, self.left, self.right))
 
     @property
     def is_empty(self):
@@ -120,7 +120,7 @@ class PackedNode(ForestNode):
             symbol = "{} ::= {}".format(rule.origin.name, ' '.join(names))
         else:
             symbol = self.s.name
-        return "{%s, %d, %d}" % (symbol, self.start.i, self.priority if self.priority is not None else 0)
+        return "{%s, %d, %d}" % (symbol, self.start, self.priority if self.priority is not None else 0)
 
 class ForestVisitor(object):
     """
diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py
index 30729ef..57aab61 100644
--- a/lark/parsers/xearley.py
+++ b/lark/parsers/xearley.py
@@ -24,7 +24,7 @@ from ..tree import Tree
 from .grammar_analysis import GrammarAnalyzer
 from ..grammar import NonTerminal, Terminal
 from .earley import ApplyCallbacks
-from .earley_common import Column, Item
+from .earley_common import Item
 from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode
 
 
@@ -44,12 +44,13 @@ class Parser:
         #  the slow 'isupper' in is_terminal.
         self.TERMINALS = { sym for r in parser_conf.rules for sym in r.expansion if sym.is_term }
         self.NON_TERMINALS = { sym for r in parser_conf.rules for sym in r.expansion if not sym.is_term }
+
         for rule in parser_conf.rules:
             self.callbacks[rule] = getattr(parser_conf.callback, rule.alias or rule.origin, None)
             self.predictions[rule.origin] = [x.rule for x in analysis.expand_rule(rule.origin)]
 
-        self.term_matcher = term_matcher
         self.forest_tree_visitor = ForestToTreeVisitor(forest_sum_visitor, self.callbacks)
+        self.term_matcher = term_matcher
 
     def parse(self, stream, start_symbol=None):
         start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
@@ -62,19 +63,20 @@ class Parser:
         # Cache for nodes & tokens created in a particular parse step.
         node_cache = {}
         token_cache = {}
+        columns = []
 
         text_line = 1
         text_column = 1
 
         def make_symbol_node(s, start, end):
-            label = (s, start.i, end.i)
+            label = (s, start, end)
             if label in node_cache:
                 node = node_cache[label]
             else:
                 node = node_cache[label] = SymbolNode(s, start, end)
             return node
 
-        def predict_and_complete(column, to_scan):
+        def predict_and_complete(i, to_scan):
             """The core Earley Predictor and Completer.
 
             At each stage of the input, we handling any completed items (things
@@ -84,15 +86,16 @@ class Parser:
             which can be added to the scan list for the next scanner cycle."""
             held_completions.clear()
 
+            column = columns[i]
             # R (items) = Ei (column.items)
-            items = deque(column.items)
+            items = deque(column)
             while items:
                 item = items.pop()    # remove an element, A say, from R
 
                 ### The Earley completer
                 if item.is_complete:   ### (item.s == string)
                     if item.node is None:
-                        item.node = make_symbol_node(item.s, item.start, column)
+                        item.node = make_symbol_node(item.s, item.start, i)
                         item.node.add_family(item.s, item.rule, item.start, None, None)
 
                     # Empty has 0 length. If we complete an empty symbol in a particular
@@ -100,19 +103,19 @@ class Parser:
                     # any predictions that result, that themselves require empty. Avoids
                     # infinite recursion on empty symbols.
                     # held_completions is 'H' in E.Scott's paper.
-                    is_empty_item = item.start.i == column.i
+                    is_empty_item = item.start == i
                     if is_empty_item:
                         held_completions[item.rule.origin] = item.node
 
-                    originators = [originator for originator in item.start.items if originator.expect is not None and originator.expect == item.s]
+                    originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s]
                     for originator in originators:
                         new_item = originator.advance()
-                        new_item.node = make_symbol_node(new_item.s, originator.start, column)
+                        new_item.node = make_symbol_node(new_item.s, originator.start, i)
                         new_item.node.add_family(new_item.s, new_item.rule, new_item.start, originator.node, item.node)
                         if new_item.expect in self.TERMINALS:
                             # Add (B :: aC.B, h, y) to Q
                             to_scan.add(new_item)
-                        elif new_item not in column.items:
+                        elif new_item not in column:
                             # Add (B :: aC.B, h, y) to Ei and R
                             column.add(new_item)
                             items.append(new_item)
@@ -121,24 +124,24 @@ class Parser:
                 elif item.expect in self.NON_TERMINALS: ### (item.s == lr0)
                     new_items = []
                     for rule in self.predictions[item.expect]:
-                        new_item = Item(rule, 0, column)
+                        new_item = Item(rule, 0, i)
                         new_items.append(new_item)
 
                     # Process any held completions (H).
                     if item.expect in held_completions:
                         new_item = item.advance()
-                        new_item.node = make_symbol_node(new_item.s, item.start, column)
+                        new_item.node = make_symbol_node(new_item.s, item.start, i)
                         new_item.node.add_family(new_item.s, new_item.rule, new_item.start, item.node, held_completions[item.expect])
                         new_items.append(new_item)
 
                     for new_item in new_items:
                         if new_item.expect in self.TERMINALS:
                             to_scan.add(new_item)
-                        elif new_item not in column.items:
+                        elif new_item not in column:
                             column.add(new_item)
                             items.append(new_item)
 
-        def scan(i, column, to_scan):
+        def scan(i, to_scan):
             """The core Earley Scanner.
 
             This is a custom implementation of the scanner that uses the
@@ -157,7 +160,7 @@ class Parser:
                 m = match(item.expect, stream, i)
                 if m:
                     t = Token(item.expect.name, m.group(0), i, text_line, text_column)
-                    delayed_matches[m.end()].append( (item, column, t) )
+                    delayed_matches[m.end()].append( (item, i, t) )
 
                     if self.complete_lex:
                         s = m.group(0)
@@ -165,7 +168,7 @@ class Parser:
                             m = match(item.expect, s[:-j])
                             if m:
                                 t = Token(item.expect.name, m.group(0), i, text_line, text_column)
-                                delayed_matches[i+m.end()].append( (item, column, t) )
+                                delayed_matches[i+m.end()].append( (item, i, t) )
 
                     # Remove any items that successfully matched in this pass from the to_scan buffer.
                     # This ensures we don't carry over tokens that already matched, if we're ignoring below.
@@ -179,13 +182,14 @@ class Parser:
                 m = match(x, stream, i)
                 if m:
                     # Carry over any items still in the scan buffer, to past the end of the ignored items.
-                    delayed_matches[m.end()].extend([(item, column, None) for item in to_scan ])
+                    delayed_matches[m.end()].extend([(item, i, None) for item in to_scan ])
 
                     # If we're ignoring up to the end of the file, # carry over the start symbol if it already completed.
-                    delayed_matches[m.end()].extend([(item, column, None) for item in column.items if item.is_complete and item.s == start_symbol])
+                    delayed_matches[m.end()].extend([(item, i, None) for item in columns[i] if item.is_complete and item.s == start_symbol])
 
-            next_set = Column(i + 1, self.FIRST)    # Ei+1
             next_to_scan = set()
+            next_set = set()
+            columns.append(next_set)
 
             ## 4) Process Tokens from delayed_matches.
             # This is the core of the Earley scanner. Create an SPPF node for each Token,
@@ -195,7 +199,8 @@ class Parser:
             for item, start, token in delayed_matches[i+1]:
                 if token is not None:
                     new_item = item.advance()
-                    new_item.node = make_symbol_node(new_item.s, new_item.start, column)
+#                   new_item.start = start # Should we update this to account for gaps due to ignores?
+                    new_item.node = make_symbol_node(new_item.s, new_item.start, i)
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token)
                 else:
                     new_item = item
@@ -212,11 +217,10 @@ class Parser:
             if not next_set and not delayed_matches and not next_to_scan:
                 raise UnexpectedCharacters(stream, i, text_line, text_column, {item.expect for item in to_scan}, set(to_scan))
 
-            return next_set, next_to_scan
+            return next_to_scan
 
         # Main loop starts
-        column0 = Column(0, self.FIRST)
-        column = column0
+        columns.append(set())
 
         ## The scan buffer. 'Q' in E.Scott's paper.
         to_scan = set()
@@ -225,38 +229,40 @@ class Parser:
         # Add predicted items to the first Earley set (for the predictor) if they
         # result in a non-terminal, or the scanner if they result in a terminal.
         for rule in self.predictions[start_symbol]:
-            item = Item(rule, 0, column0)
+            item = Item(rule, 0, 0)
             if item.expect in self.TERMINALS:
                 to_scan.add(item)
             else:
-                column.add(item)
+                columns[0].add(item)
 
         ## The main Earley loop.
         # Run the Prediction/Completion cycle for any Items in the current Earley set.
         # Completions will be added to the SPPF tree, and predictions will be recursively
         # processed down to terminals/empty nodes to be added to the scanner for the next
         # step.
-        for i, token in enumerate(stream):
-            predict_and_complete(column, to_scan)
+        i = 0
+        for token in stream:
+            predict_and_complete(i, to_scan)
 
             # Clear the node_cache and token_cache, which are only relevant for each
             # step in the Earley pass.
             node_cache.clear()
             token_cache.clear()
-            column, to_scan = scan(i, column, to_scan)
+            to_scan = scan(i, to_scan)
 
             if token == '\n':
                 text_line += 1
                 text_column = 1
             else:
                 text_column += 1
+            i += 1
 
-        predict_and_complete(column, to_scan)
+        predict_and_complete(i, to_scan)
 
         ## Column is now the final column in the parse. If the parse was successful, the start
         # symbol should have been completed in the last step of the Earley cycle, and will be in
         # this column. Find the item for the start_symbol, which is the root of the SPPF tree.
-        solutions = [n.node for n in column.items if n.is_complete and n.node is not None and n.s == start_symbol and n.start is column0]
+        solutions = [n.node for n in columns[i] if n.is_complete and n.node is not None and n.s == start_symbol and n.start == 0]
 
         if not solutions:
             expected_tokens = [t.expect for t in to_scan]

From 637f12110979999afc3bab3afbd514ce5e651bd2 Mon Sep 17 00:00:00 2001
From: night199uk <night199uk@hermitcrabslab.com>
Date: Fri, 14 Dec 2018 12:49:29 +0100
Subject: [PATCH 3/4] Cleanup unused Derivation

---
 lark/parsers/earley_common.py | 17 -----------------
 lark/parsers/earley_forest.py |  1 -
 2 files changed, 18 deletions(-)

diff --git a/lark/parsers/earley_common.py b/lark/parsers/earley_common.py
index 74dd388..e13c7c3 100644
--- a/lark/parsers/earley_common.py
+++ b/lark/parsers/earley_common.py
@@ -13,23 +13,6 @@
 # Author: Erez Shinan (2017)
 # Email : erezshin@gmail.com
 
-## for recursive repr
-from ..tree import Tree
-
-class Derivation(Tree):
-    def __init__(self, rule, children = None):
-        Tree.__init__(self, 'drv', children if children is not None else [])
-        self.meta.rule = rule
-        self._hash = None
-
-    def __repr__(self, indent = 0):
-        return 'Derivation(%s, %s, %s)' % (self.data, self.rule.origin, '...')
-
-    def __hash__(self):
-        if self._hash is None:
-            self._hash = Tree.__hash__(self)
-        return self._hash
-
 class Item(object):
     "An Earley Item, the atom of the algorithm."
 
diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py
index 730ebe1..f6bea75 100644
--- a/lark/parsers/earley_forest.py
+++ b/lark/parsers/earley_forest.py
@@ -13,7 +13,6 @@ from ..exceptions import ParseError
 from ..lexer import Token
 from ..utils import Str
 from ..grammar import NonTerminal, Terminal
-from .earley_common import Derivation
 
 from collections import deque
 from importlib import import_module

From 04d90fa9165741333c2e394592a64e1c3966aa14 Mon Sep 17 00:00:00 2001
From: night199uk <night199uk@hermitcrabslab.com>
Date: Tue, 18 Dec 2018 05:57:57 +0100
Subject: [PATCH 4/4] Implement Joop Leo's optimizations for right recursion
 performance

---
 lark/parsers/earley.py        | 137 +++++++++++++++++++++++++++-------
 lark/parsers/earley_common.py |  38 +++++++++-
 lark/parsers/earley_forest.py |  67 +++++++++++------
 lark/parsers/xearley.py       | 137 +++++++++++++++++++++++++++-------
 4 files changed, 303 insertions(+), 76 deletions(-)

diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py
index 7600915..66ab903 100644
--- a/lark/parsers/earley.py
+++ b/lark/parsers/earley.py
@@ -16,7 +16,7 @@ from ..visitors import Transformer_InPlace, v_args
 from ..exceptions import ParseError, UnexpectedToken
 from .grammar_analysis import GrammarAnalyzer
 from ..grammar import NonTerminal
-from .earley_common import Item
+from .earley_common import Item, TransitiveItem
 from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode
 
 from collections import deque, defaultdict
@@ -28,6 +28,7 @@ class Parser:
         self.resolve_ambiguity = resolve_ambiguity
 
         self.FIRST = analysis.FIRST
+        self.NULLABLE = analysis.NULLABLE
         self.callbacks = {}
         self.predictions = {}
 
@@ -56,14 +57,68 @@ class Parser:
         node_cache = {}
         token_cache = {}
         columns = []
-
-        def make_symbol_node(s, start, end):
-            label = (s, start, end)
-            if label in node_cache:
-                node = node_cache[label]
+        transitives = []
+
+        def is_quasi_complete(item):
+            if item.is_complete:
+                return True
+
+            quasi = item.advance()
+            while not quasi.is_complete:
+                symbol = quasi.expect
+                if symbol not in self.NULLABLE:
+                    return False
+                if quasi.rule.origin == start_symbol and symbol == start_symbol:
+                    return False
+                quasi = quasi.advance()
+            return True
+
+        def create_leo_transitives(item, trule, previous, visited = None):
+            if visited is None:
+                visited = set()
+
+            if item.rule.origin in transitives[item.start]:
+                previous = trule = transitives[item.start][item.rule.origin]
+                return trule, previous
+
+            is_empty_rule = not self.FIRST[item.rule.origin]
+            if is_empty_rule:
+                return trule, previous
+
+            originator = None
+            for key in columns[item.start]:
+                if key.expect is not None and key.expect == item.rule.origin:
+                    if originator is not None:
+                        return trule, previous
+                    originator = key
+
+            if originator is None:
+                return trule, previous
+
+            if originator in visited:
+                return trule, previous
+
+            visited.add(originator)
+            if not is_quasi_complete(originator):
+                return trule, previous
+
+            trule = originator.advance()
+            if originator.start != item.start:
+                visited.clear()
+
+            trule, previous = create_leo_transitives(originator, trule, previous, visited)
+            if trule is None:
+                return trule, previous
+
+            titem = None
+            if previous is not None:
+                titem = TransitiveItem(item.rule.origin, trule, originator, previous.column)
+                previous.next_titem = titem
             else:
-                node = node_cache[label] = SymbolNode(s, start, end)
-            return node
+                titem = TransitiveItem(item.rule.origin, trule, originator, item.start)
+
+            previous = transitives[item.start][item.rule.origin] = titem
+            return trule, previous
 
         def predict_and_complete(i, to_scan):
             """The core Earley Predictor and Completer.
@@ -84,23 +139,26 @@ class Parser:
                 ### The Earley completer
                 if item.is_complete:   ### (item.s == string)
                     if item.node is None:
-                        item.node = make_symbol_node(item.s, item.start, i)
+                        label = (item.s, item.start, i)
+                        item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
                         item.node.add_family(item.s, item.rule, item.start, None, None)
 
-                    # Empty has 0 length. If we complete an empty symbol in a particular
-                    # parse step, we need to be able to use that same empty symbol to complete
-                    # any predictions that result, that themselves require empty. Avoids
-                    # infinite recursion on empty symbols.
-                    # held_completions is 'H' in E.Scott's paper.
-                    is_empty_item = item.start == i
-                    if is_empty_item:
-                        held_completions[item.rule.origin] = item.node
-
-                    originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s]
-                    for originator in originators:
-                        new_item = originator.advance()
-                        new_item.node = make_symbol_node(new_item.s, originator.start, i)
-                        new_item.node.add_family(new_item.s, new_item.rule, new_item.start, originator.node, item.node)
+                    create_leo_transitives(item, None, None)
+
+                    ###R Joop Leo right recursion Completer
+                    if item.rule.origin in transitives[item.start]:
+                        transitive = transitives[item.start][item.s]
+                        if transitive.previous in transitives[transitive.column]:
+                            root_transitive = transitives[transitive.column][transitive.previous]
+                        else:
+                            root_transitive = transitive
+
+                        label = (root_transitive.s, root_transitive.start, i)
+                        node = vn = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                        vn.add_path(root_transitive, item.node)
+
+                        new_item = Item(transitive.rule, transitive.ptr, transitive.start)
+                        new_item.node = vn
                         if new_item.expect in self.TERMINALS:
                             # Add (B :: aC.B, h, y) to Q
                             to_scan.add(new_item)
@@ -108,6 +166,30 @@ class Parser:
                             # Add (B :: aC.B, h, y) to Ei and R
                             column.add(new_item)
                             items.append(new_item)
+                    ###R Regular Earley completer
+                    else:
+                        # Empty has 0 length. If we complete an empty symbol in a particular
+                        # parse step, we need to be able to use that same empty symbol to complete
+                        # any predictions that result, that themselves require empty. Avoids
+                        # infinite recursion on empty symbols.
+                        # held_completions is 'H' in E.Scott's paper.
+                        is_empty_item = item.start == i
+                        if is_empty_item:
+                            held_completions[item.rule.origin] = item.node
+
+                        originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s]
+                        for originator in originators:
+                            new_item = originator.advance()
+                            label = (new_item.s, originator.start, i)
+                            new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                            new_item.node.add_family(new_item.s, new_item.rule, i, originator.node, item.node)
+                            if new_item.expect in self.TERMINALS:
+                                # Add (B :: aC.B, h, y) to Q
+                                to_scan.add(new_item)
+                            elif new_item not in column:
+                                # Add (B :: aC.B, h, y) to Ei and R
+                                column.add(new_item)
+                                items.append(new_item)
 
                 ### The Earley predictor
                 elif item.expect in self.NON_TERMINALS: ### (item.s == lr0)
@@ -119,7 +201,8 @@ class Parser:
                     # Process any held completions (H).
                     if item.expect in held_completions:
                         new_item = item.advance()
-                        new_item.node = make_symbol_node(new_item.s, item.start, i)
+                        label = (new_item.s, item.start, i)
+                        new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
                         new_item.node.add_family(new_item.s, new_item.rule, new_item.start, item.node, held_completions[item.expect])
                         new_items.append(new_item)
 
@@ -141,11 +224,14 @@ class Parser:
             next_to_scan = set()
             next_set = set()
             columns.append(next_set)
+            next_transitives = dict()
+            transitives.append(next_transitives)
 
             for item in set(to_scan):
                 if match(item.expect, token):
                     new_item = item.advance()
-                    new_item.node = make_symbol_node(new_item.s, new_item.start, i)
+                    label = (new_item.s, new_item.start, i)
+                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token)
 
                     if new_item.expect in self.TERMINALS:
@@ -163,6 +249,7 @@ class Parser:
 
         # Main loop starts
         columns.append(set())
+        transitives.append(dict())
 
         ## The scan buffer. 'Q' in E.Scott's paper.
         to_scan = set()
diff --git a/lark/parsers/earley_common.py b/lark/parsers/earley_common.py
index e13c7c3..6bd614b 100644
--- a/lark/parsers/earley_common.py
+++ b/lark/parsers/earley_common.py
@@ -13,12 +13,13 @@
 # Author: Erez Shinan (2017)
 # Email : erezshin@gmail.com
 
+from ..grammar import NonTerminal, Terminal
+
 class Item(object):
     "An Earley Item, the atom of the algorithm."
 
-    __slots__ = ('s', 'rule', 'ptr', 'start', 'is_complete', 'expect', 'node', '_hash')
+    __slots__ = ('s', 'rule', 'ptr', 'start', 'is_complete', 'expect', 'previous', 'node', '_hash')
     def __init__(self, rule, ptr, start):
-        assert isinstance(start, int), "start is not an int"
         self.is_complete = len(rule.expansion) == ptr
         self.rule = rule    # rule
         self.ptr = ptr      # ptr
@@ -27,13 +28,15 @@ class Item(object):
         if self.is_complete:
             self.s = rule.origin
             self.expect = None
+            self.previous = rule.expansion[ptr - 1] if ptr > 0 and len(rule.expansion) else None
         else:
             self.s = (rule, ptr)
             self.expect = rule.expansion[ptr]
+            self.previous = rule.expansion[ptr - 1] if ptr > 0 and len(rule.expansion) else None
         self._hash = hash((self.s, self.start))
 
     def advance(self):
-        return self.__class__(self.rule, self.ptr + 1, self.start)
+        return Item(self.rule, self.ptr + 1, self.start)
 
     def __eq__(self, other):
         return self is other or (self.s == other.s and self.start == other.start)
@@ -42,4 +45,31 @@ class Item(object):
         return self._hash
 
     def __repr__(self):
-        return '%s (%d)' % (self.s if self.is_complete else self.rule.origin, self.start)
+        before = ( expansion.name for expansion in self.rule.expansion[:self.ptr] )
+        after = ( expansion.name for expansion in self.rule.expansion[self.ptr:] )
+        symbol = "{} ::= {}* {}".format(self.rule.origin.name, ' '.join(before), ' '.join(after))
+        return '%s (%d)' % (symbol, self.start)
+
+
+class TransitiveItem(Item):
+    __slots__ = ('recognized', 'reduction', 'column', 'next_titem')
+    def __init__(self, recognized, trule, originator, start):
+        super(TransitiveItem, self).__init__(trule.rule, trule.ptr, trule.start)
+        self.recognized = recognized
+        self.reduction = originator
+        self.column = start
+        self.next_titem = None
+        self._hash = hash((self.s, self.start, self.recognized))
+
+    def __eq__(self, other):
+        if not isinstance(other, TransitiveItem):
+            return False
+        return self is other or (type(self.s) == type(other.s) and self.s == other.s and self.start == other.start and self.recognized == other.recognized)
+
+    def __hash__(self):
+        return self._hash
+
+    def __repr__(self):
+        before = ( expansion.name for expansion in self.rule.expansion[:self.ptr] )
+        after = ( expansion.name for expansion in self.rule.expansion[self.ptr:] )
+        return '{} : {} -> {}* {} ({}, {})'.format(self.recognized.name, self.rule.origin.name, ' '.join(before), ' '.join(after), self.column, self.start)
diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py
index f6bea75..dda2dcb 100644
--- a/lark/parsers/earley_forest.py
+++ b/lark/parsers/earley_forest.py
@@ -12,7 +12,7 @@ from ..tree import Tree
 from ..exceptions import ParseError
 from ..lexer import Token
 from ..utils import Str
-from ..grammar import NonTerminal, Terminal
+from ..grammar import NonTerminal, Terminal, Symbol
 
 from collections import deque
 from importlib import import_module
@@ -34,42 +34,65 @@ class SymbolNode(ForestNode):
 
     Hence a Symbol Node with a single child is unambiguous.
     """
-    __slots__ = ('s', 'start', 'end', 'children', 'priority', 'is_intermediate')
+    __slots__ = ('s', 'start', 'end', '_children', 'paths', 'paths_loaded', 'priority', 'is_intermediate', '_hash')
     def __init__(self, s, start, end):
         self.s = s
         self.start = start
         self.end = end
-        self.children = set()
+        self._children = set()
+        self.paths = set()
+        self.paths_loaded = False
         self.priority = None
         self.is_intermediate = isinstance(s, tuple)
+        self._hash = hash((self.s, self.start, self.end))
 
     def add_family(self, lr0, rule, start, left, right):
-        self.children.add(PackedNode(self, lr0, rule, start, left, right))
+        self._children.add(PackedNode(self, lr0, rule, start, left, right))
+
+    def add_path(self, transitive, node):
+        self.paths.add((transitive, node))
+
+    def load_paths(self):
+        for transitive, node in self.paths:
+            if transitive.next_titem is not None:
+                vn = SymbolNode(transitive.next_titem.s, transitive.next_titem.start, self.end)
+                vn.add_path(transitive.next_titem, node)
+                self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, vn)
+            else:
+                self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, node)
+        self.paths_loaded = True
 
     @property
     def is_ambiguous(self):
         return len(self.children) > 1
 
+    @property
+    def children(self):
+        if not self.paths_loaded:
+            self.load_paths()
+        return self._children
+
     def __iter__(self):
-        return iter(self.children)
+        return iter(self._children)
 
     def __eq__(self, other):
         if not isinstance(other, SymbolNode):
             return False
-        return self is other or (self.s == other.s and self.start == other.start and self.end is other.end)
+        return self is other or (type(self.s) == type(other.s) and self.s == other.s and self.start == other.start and self.end is other.end)
 
     def __hash__(self):
-        return hash((self.s, self.start, self.end))
+        return self._hash
 
     def __repr__(self):
         if self.is_intermediate:
             rule = self.s[0]
             ptr = self.s[1]
-            names = [ "{}*".format(expansion.name) if index == ptr else expansion.name for index, expansion in enumerate(rule.expansion) ]
-            symbol = "{} ::= {}".format(rule.origin.name, ' '.join(names))
+            before = ( expansion.name for expansion in rule.expansion[:ptr] )
+            after = ( expansion.name for expansion in rule.expansion[ptr:] )
+            symbol = "{} ::= {}* {}".format(rule.origin.name, ' '.join(before), ' '.join(after))
         else:
             symbol = self.s.name
-        return "(%s, %d, %d, %d)" % (symbol, self.start, self.end, self.priority if self.priority is not None else 0)
+        return "({}, {}, {}, {})".format(symbol, self.start, self.end, self.priority if self.priority is not None else 0)
 
 class PackedNode(ForestNode):
     """
@@ -115,11 +138,12 @@ class PackedNode(ForestNode):
         if isinstance(self.s, tuple):
             rule = self.s[0]
             ptr = self.s[1]
-            names = [ "{}*".format(expansion.name) if index == ptr else expansion.name for index, expansion in enumerate(rule.expansion) ]
-            symbol = "{} ::= {}".format(rule.origin.name, ' '.join(names))
+            before = ( expansion.name for expansion in rule.expansion[:ptr] )
+            after = ( expansion.name for expansion in rule.expansion[ptr:] )
+            symbol = "{} ::= {}* {}".format(rule.origin.name, ' '.join(before), ' '.join(after))
         else:
             symbol = self.s.name
-        return "{%s, %d, %d}" % (symbol, self.start, self.priority if self.priority is not None else 0)
+        return "({}, {}, {})".format(symbol, self.start, self.priority)
 
 class ForestVisitor(object):
     """
@@ -182,8 +206,8 @@ class ForestVisitor(object):
 
             current_id = id(current)
             if current_id in visiting:
-                if isinstance(current, PackedNode): vpno(current)
-                else:                               vsno(current)
+                if isinstance(current, PackedNode):    vpno(current)
+                else:                                  vsno(current)
                 input_stack.pop()
                 visiting.remove(current_id)
                 continue
@@ -226,7 +250,7 @@ class ForestSumVisitor(ForestVisitor):
 
     def visit_symbol_node_out(self, node):
         node.priority = max(child.priority for child in node.children)
-        node.children = sorted(node.children, reverse = True)
+        node._children = sorted(node.children, reverse = True)
 
 class ForestAntiscoreSumVisitor(ForestSumVisitor):
     """
@@ -240,7 +264,7 @@ class ForestAntiscoreSumVisitor(ForestSumVisitor):
     """
     def visit_symbol_node_out(self, node):
         node.priority = min(child.priority for child in node.children)
-        node.children = sorted(node.children, key=AntiscoreSumComparator, reverse = True)
+        node._children = sorted(node.children, key=AntiscoreSumComparator, reverse = True)
 
 class AntiscoreSumComparator(object):
     """
@@ -342,7 +366,7 @@ class ForestToAmbiguousTreeVisitor(ForestVisitor):
         return iter(node.children)
 
     def visit_symbol_node_out(self, node):
-        if node.is_ambiguous:
+        if not node.is_intermediate and node.is_ambiguous:
             result = self.output_stack.pop()
             if self.output_stack:
                 self.output_stack[-1].children.append(result)
@@ -386,8 +410,8 @@ class ForestToPyDotVisitor(ForestVisitor):
         graph_node_id = str(id(node))
         graph_node_label = "\"{}\"".format(node.value.replace('"', '\\"'))
         graph_node_color = 0x808080
-        graph_node_style = "filled"
-        graph_node_shape = "polygon"
+        graph_node_style = "\"filled,rounded\""
+        graph_node_shape = "diamond"
         graph_node = self.pydot.Node(graph_node_id, style=graph_node_style, fillcolor="#{:06x}".format(graph_node_color), shape=graph_node_shape, label=graph_node_label)
         self.graph.add_node(graph_node)
 
@@ -422,7 +446,7 @@ class ForestToPyDotVisitor(ForestVisitor):
         graph_node_id = str(id(node))
         graph_node_label = repr(node)
         graph_node_color = 0x808080
-        graph_node_style = "filled"
+        graph_node_style = "\"filled\""
         if node.is_intermediate:
             graph_node_shape = "ellipse"
         else:
@@ -438,4 +462,3 @@ class ForestToPyDotVisitor(ForestVisitor):
             child_graph_node_id = str(id(child))
             child_graph_node = self.graph.get_node(child_graph_node_id)[0]
             self.graph.add_edge(self.pydot.Edge(graph_node, child_graph_node))
-
diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py
index 57aab61..acd5d25 100644
--- a/lark/parsers/xearley.py
+++ b/lark/parsers/xearley.py
@@ -24,7 +24,7 @@ from ..tree import Tree
 from .grammar_analysis import GrammarAnalyzer
 from ..grammar import NonTerminal, Terminal
 from .earley import ApplyCallbacks
-from .earley_common import Item
+from .earley_common import Item, TransitiveItem
 from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode
 
 
@@ -37,6 +37,7 @@ class Parser:
         self.complete_lex = complete_lex
 
         self.FIRST = analysis.FIRST
+        self.NULLABLE = analysis.NULLABLE
         self.callbacks = {}
         self.predictions = {}
 
@@ -64,17 +65,71 @@ class Parser:
         node_cache = {}
         token_cache = {}
         columns = []
+        transitives = []
 
         text_line = 1
         text_column = 1
 
-        def make_symbol_node(s, start, end):
-            label = (s, start, end)
-            if label in node_cache:
-                node = node_cache[label]
+        def is_quasi_complete(item):
+            if item.is_complete:
+                return True
+
+            quasi = item.advance()
+            while not quasi.is_complete:
+                symbol = quasi.expect
+                if symbol not in self.NULLABLE:
+                    return False
+                if quasi.rule.origin == start_symbol and symbol == start_symbol:
+                    return False
+                quasi = quasi.advance()
+            return True
+
+        def create_leo_transitives(item, trule, previous, visited = None):
+            if visited is None:
+                visited = set()
+
+            if item.rule.origin in transitives[item.start]:
+                previous = trule = transitives[item.start][item.rule.origin]
+                return trule, previous
+
+            is_empty_rule = not self.FIRST[item.rule.origin]
+            if is_empty_rule:
+                return trule, previous
+
+            originator = None
+            for key in columns[item.start]:
+                if key.expect is not None and key.expect == item.rule.origin:
+                    if originator is not None:
+                        return trule, previous
+                    originator = key
+
+            if originator is None:
+                return trule, previous
+
+            if originator in visited:
+                return trule, previous
+
+            visited.add(originator)
+            if not is_quasi_complete(originator):
+                return trule, previous
+
+            trule = originator.advance()
+            if originator.start != item.start:
+                visited.clear()
+
+            trule, previous = create_leo_transitives(originator, trule, previous, visited)
+            if trule is None:
+                return trule, previous
+
+            titem = None
+            if previous is not None:
+                titem = TransitiveItem(item.rule.origin, trule, originator, previous.column)
+                previous.next_titem = titem
             else:
-                node = node_cache[label] = SymbolNode(s, start, end)
-            return node
+                titem = TransitiveItem(item.rule.origin, trule, originator, item.start)
+
+            previous = transitives[item.start][item.rule.origin] = titem
+            return trule, previous
 
         def predict_and_complete(i, to_scan):
             """The core Earley Predictor and Completer.
@@ -95,23 +150,26 @@ class Parser:
                 ### The Earley completer
                 if item.is_complete:   ### (item.s == string)
                     if item.node is None:
-                        item.node = make_symbol_node(item.s, item.start, i)
+                        label = (item.s, item.start, i)
+                        item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
                         item.node.add_family(item.s, item.rule, item.start, None, None)
 
-                    # Empty has 0 length. If we complete an empty symbol in a particular
-                    # parse step, we need to be able to use that same empty symbol to complete
-                    # any predictions that result, that themselves require empty. Avoids
-                    # infinite recursion on empty symbols.
-                    # held_completions is 'H' in E.Scott's paper.
-                    is_empty_item = item.start == i
-                    if is_empty_item:
-                        held_completions[item.rule.origin] = item.node
-
-                    originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s]
-                    for originator in originators:
-                        new_item = originator.advance()
-                        new_item.node = make_symbol_node(new_item.s, originator.start, i)
-                        new_item.node.add_family(new_item.s, new_item.rule, new_item.start, originator.node, item.node)
+                    create_leo_transitives(item, None, None)
+
+                    ###R Joop Leo right recursion Completer
+                    if item.rule.origin in transitives[item.start]:
+                        transitive = transitives[item.start][item.s]
+                        if transitive.previous in transitives[transitive.column]:
+                            root_transitive = transitives[transitive.column][transitive.previous]
+                        else:
+                            root_transitive = transitive
+
+                        label = (root_transitive.s, root_transitive.start, i)
+                        node = vn = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                        vn.add_path(root_transitive, item.node)
+
+                        new_item = Item(transitive.rule, transitive.ptr, transitive.start)
+                        new_item.node = vn
                         if new_item.expect in self.TERMINALS:
                             # Add (B :: aC.B, h, y) to Q
                             to_scan.add(new_item)
@@ -119,6 +177,30 @@ class Parser:
                             # Add (B :: aC.B, h, y) to Ei and R
                             column.add(new_item)
                             items.append(new_item)
+                    ###R Regular Earley completer
+                    else:
+                        # Empty has 0 length. If we complete an empty symbol in a particular
+                        # parse step, we need to be able to use that same empty symbol to complete
+                        # any predictions that result, that themselves require empty. Avoids
+                        # infinite recursion on empty symbols.
+                        # held_completions is 'H' in E.Scott's paper.
+                        is_empty_item = item.start == i
+                        if is_empty_item:
+                            held_completions[item.rule.origin] = item.node
+
+                        originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s]
+                        for originator in originators:
+                            new_item = originator.advance()
+                            label = (new_item.s, originator.start, i)
+                            new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                            new_item.node.add_family(new_item.s, new_item.rule, i, originator.node, item.node)
+                            if new_item.expect in self.TERMINALS:
+                                # Add (B :: aC.B, h, y) to Q
+                                to_scan.add(new_item)
+                            elif new_item not in column:
+                                # Add (B :: aC.B, h, y) to Ei and R
+                                column.add(new_item)
+                                items.append(new_item)
 
                 ### The Earley predictor
                 elif item.expect in self.NON_TERMINALS: ### (item.s == lr0)
@@ -130,7 +212,8 @@ class Parser:
                     # Process any held completions (H).
                     if item.expect in held_completions:
                         new_item = item.advance()
-                        new_item.node = make_symbol_node(new_item.s, item.start, i)
+                        label = (new_item.s, item.start, i)
+                        new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
                         new_item.node.add_family(new_item.s, new_item.rule, new_item.start, item.node, held_completions[item.expect])
                         new_items.append(new_item)
 
@@ -190,6 +273,8 @@ class Parser:
             next_to_scan = set()
             next_set = set()
             columns.append(next_set)
+            next_transitives = dict()
+            transitives.append(next_transitives)
 
             ## 4) Process Tokens from delayed_matches.
             # This is the core of the Earley scanner. Create an SPPF node for each Token,
@@ -199,8 +284,8 @@ class Parser:
             for item, start, token in delayed_matches[i+1]:
                 if token is not None:
                     new_item = item.advance()
-#                   new_item.start = start # Should we update this to account for gaps due to ignores?
-                    new_item.node = make_symbol_node(new_item.s, new_item.start, i)
+                    label = (new_item.s, new_item.start, i)
+                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token)
                 else:
                     new_item = item
@@ -221,6 +306,7 @@ class Parser:
 
         # Main loop starts
         columns.append(set())
+        transitives.append(dict())
 
         ## The scan buffer. 'Q' in E.Scott's paper.
         to_scan = set()
@@ -248,6 +334,7 @@ class Parser:
             # step in the Earley pass.
             node_cache.clear()
             token_cache.clear()
+            node_cache.clear()
             to_scan = scan(i, to_scan)
 
             if token == '\n':