Merge branch 'evilnose-copy-fix'

lark/lexer.py

@@ -186,6 +186,12 @@ class LineCounter:
         self.column = 1
         self.line_start_pos = 0
 
+    def __eq__(self, other):
+        if not isinstance(other, LineCounter):
+            return NotImplemented
+
+        return self.char_pos == other.char_pos and self.newline_char == other.newline_char
+
     def feed(self, token, test_newline=True):
         """Consume a token and calculate the new line & column.
 
@@ -397,7 +403,7 @@ class TraditionalLexer(Lexer):
         raise EOFError(self)
 
 
-class LexerState:
+class LexerState(object):
     __slots__ = 'text', 'line_ctr', 'last_token'
 
     def __init__(self, text, line_ctr, last_token=None):
@@ -405,6 +411,12 @@ class LexerState:
         self.line_ctr = line_ctr
         self.last_token = last_token
 
+    def __eq__(self, other):
+        if not isinstance(other, LexerState):
+            return NotImplemented
+
+        return self.text is other.text and self.line_ctr == other.line_ctr and self.last_token == other.last_token
+
     def __copy__(self):
         return type(self)(self.text, copy(self.line_ctr), self.last_token)
 
@@ -456,7 +468,7 @@ class ContextualLexer(Lexer):
             except UnexpectedCharacters:
                 raise e  # Raise the original UnexpectedCharacters. The root lexer raises it with the wrong expected set.
 
-class LexerThread:
+class LexerThread(object):
     """A thread that ties a lexer instance and a lexer state, to be used by the parser"""
 
     def __init__(self, lexer, text):
@@ -465,4 +477,10 @@ class LexerThread:
 
     def lex(self, parser_state):
         return self.lexer.lex(self.state, parser_state)
+
+    def __copy__(self):
+        copied = object.__new__(LexerThread)
+        copied.lexer = self.lexer
+        copied.state = copy(self.state)
+        return copied
 ###}

lark/parsers/lalr_parser.py

@@ -94,8 +94,8 @@ class ParserState(object):
     # Necessary for match_examples() to work
     def __eq__(self, other):
         if not isinstance(other, ParserState):
-            return False
-        return self.position == other.position
+            return NotImplemented
+        return len(self.state_stack) == len(other.state_stack) and self.position == other.position
 
     def __copy__(self):
         return type(self)(

tests/test_parser.py

@@ -2408,9 +2408,18 @@ def _make_parser_test(LEXER, PARSER):
                     # Skip comma
                     return True
                 elif e.token.type == 'SIGNED_NUMBER':
+                    # Make a copy and ensure it is properly made
+                    puppet_copy = e.puppet.copy()
+                    assert puppet_copy.parser_state == e.puppet.parser_state
+                    assert puppet_copy.lexer_state.state == e.puppet.lexer_state.state
+                    assert puppet_copy.parser_state is not e.puppet.parser_state
+                    assert puppet_copy.lexer_state.state is not e.puppet.lexer_state.state
+                    assert puppet_copy.lexer_state.state.line_ctr is not e.puppet.lexer_state.state.line_ctr
+
                     # Try to feed a comma and retry the number
                     e.puppet.feed_token(Token('COMMA', ','))
                     e.puppet.feed_token(e.token)
+
                     return True
 
                 # Unhandled error. Will stop parse and raise exception