BUGFIX: Tokens of different type were equal, causing disambiguation errors (Issue #21)

lark/lexer.py

@@ -40,6 +40,14 @@ class Token(Str):
     def __deepcopy__(self, memo):
         return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)
 
+    def __eq__(self, other):
+        if isinstance(other, Token) and self.type != other.type:
+            return False
+        
+        return Str.__eq__(self, other)
+
+    __hash__ = Str.__hash__
+
 class Regex:
     def __init__(self, pattern, flags=()):
         self.pattern = pattern

lark/parsers/resolve_ambig.py

@@ -84,23 +84,12 @@ def _antiscore_sum_drv(tree):
     if not isinstance(tree, Tree):
         return 0
 
-    # XXX These artifacts can appear due to imperfections in the ordering of Visitor_NoRecurse,
-    #     when confronted with duplicate (same-id) nodes. Fixing this ordering is possible, but would be
-    #     computationally inefficient. So we handle it here.
-    if tree.data == '_ambig':
-        _antiscore_sum_resolve_ambig(tree)
+    assert tree.data != '_ambig'
 
-    try:
-        priority = tree.rule.options.priority
-    except AttributeError:
-        # Probably trees that don't take part in this parse (better way to distinguish?)
-        priority = None
-
-    return (priority or 0) + sum(map(_antiscore_sum_drv, tree.children), 0)
+    return _sum_priority(tree)
 
 def _antiscore_sum_resolve_ambig(tree):
     assert tree.data == '_ambig'
-
     best = min(tree.children, key=_antiscore_sum_drv)
     assert best.data == 'drv'
     tree.set('drv', best.children)