From 7be2c046376300f257a89c83de8cbfab990560c0 Mon Sep 17 00:00:00 2001
From: night199uk <night199uk@hermitcrabslab.com>
Date: Fri, 16 Mar 2018 03:33:03 +0100
Subject: [PATCH] Ensure Tokens can be pickled correctly

---
 lark/lexer.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/lark/lexer.py b/lark/lexer.py
index e48a970..bcf09f1 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -26,18 +26,21 @@ class UnexpectedInput(LexError):
 
 class Token(Str):
     def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
-        inst = Str.__new__(cls, value)
-        inst.type = type_
-        inst.pos_in_stream = pos_in_stream
-        inst.value = value
-        inst.line = line
-        inst.column = column
-        return inst
+        self = super(Token, cls).__new__(cls, value)
+        self.type = type_
+        self.pos_in_stream = pos_in_stream
+        self.value = value
+        self.line = line
+        self.column = column
+        return self
 
     @classmethod
     def new_borrow_pos(cls, type_, value, borrow_t):
         return cls(type_, value, borrow_t.pos_in_stream, line=borrow_t.line, column=borrow_t.column)
 
+    def __reduce__(self):
+        return (self.__class__, (self.type, self.pos_in_stream, self.value, self.line, self.column, ))
+
     def __repr__(self):
         return 'Token(%s, %r)' % (self.type, self.value)