From b3eb2a31201b3aba6626502ee771885a556438a6 Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Sat, 21 Nov 2020 22:51:22 +0200 Subject: [PATCH] Add token_history to UnexpectedToken --- lark/exceptions.py | 6 +++++- lark/lexer.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lark/exceptions.py b/lark/exceptions.py index 8444a65..ed7b9c7 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -147,7 +147,7 @@ class UnexpectedToken(ParseError, UnexpectedInput): see: :ref:`ParserPuppet`. """ - def __init__(self, token, expected, considered_rules=None, state=None, puppet=None): + def __init__(self, token, expected, considered_rules=None, state=None, puppet=None, token_history=None): self.line = getattr(token, 'line', '?') self.column = getattr(token, 'column', '?') self.pos_in_stream = getattr(token, 'pos_in_stream', None) @@ -157,6 +157,7 @@ class UnexpectedToken(ParseError, UnexpectedInput): self.expected = expected # XXX deprecate? `accepts` is better self.considered_rules = considered_rules self.puppet = puppet + self.token_history = token_history # TODO Only calculate `accepts()` when we need to display it to the user # This will improve performance when doing automatic error handling @@ -166,6 +167,9 @@ class UnexpectedToken(ParseError, UnexpectedInput): "Expected one of: \n\t* %s\n" % (token, self.line, self.column, '\n\t* '.join(self.accepts or self.expected))) + if self.token_history: + message += "Previous tokens: %r\n" % token_history + super(UnexpectedToken, self).__init__(message) diff --git a/lark/lexer.py b/lark/lexer.py index 6d69ec9..8be8acd 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -436,7 +436,7 @@ class ContextualLexer(Lexer): # In the contextual lexer, UnexpectedCharacters can mean that the terminal is defined, but not in the current context. # This tests the input against the global context, to provide a nicer error. token = self.root_lexer.next_token(lexer_state, parser_state) - raise UnexpectedToken(token, e.allowed, state=parser_state.position) + raise UnexpectedToken(token, e.allowed, state=parser_state.position, token_history=[lexer_state.last_token]) class LexerThread: