From a9129f33c27b475de703ebe3706c09ac769e7fd4 Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Mon, 13 Sep 2021 11:53:10 +0100 Subject: [PATCH] Remove Py2-related unicode patch --- lark/lexer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lark/lexer.py b/lark/lexer.py index afdbade..91c523a 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -157,12 +157,7 @@ class Token(str): end_pos: int def __new__(cls, type_, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): - try: - inst = super(Token, cls).__new__(cls, value) - except UnicodeDecodeError: - value = value.decode('latin1') - inst = super(Token, cls).__new__(cls, value) - + inst = super(Token, cls).__new__(cls, value) inst.type = type_ inst.start_pos = start_pos inst.value = value