diff --git a/lark/exceptions.py b/lark/exceptions.py index cfb988f..9fee01e 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -50,21 +50,22 @@ class UnexpectedInput(LarkError): class UnexpectedCharacters(LexError, UnexpectedInput): def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None): - context = seq[lex_pos:lex_pos+10] - message = "No token defined for '%s' in %r at line %d col %d" % (seq[lex_pos], context, line, column) - if allowed: - message += '\n\nExpecting: %s\n' % allowed - - super(UnexpectedCharacters, self).__init__(message) + message = "No terminal defined for '%s' at line %d col %d" % (seq[lex_pos], line, column) self.line = line self.column = column - self.context = context self.allowed = allowed self.considered_tokens = considered_tokens self.pos_in_stream = lex_pos self.state = state + message += '\n\n' + self.get_context(seq) + if allowed: + message += '\nExpecting: %s\n' % allowed + + super(UnexpectedCharacters, self).__init__(message) + + class UnexpectedToken(ParseError, UnexpectedInput): def __init__(self, token, expected, considered_rules=None, state=None): diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 0d9bdc9..73082df 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -311,7 +311,7 @@ class PrepareAnonTerminals(Transformer_InPlace): self.token_reverse[p] = tokendef self.tokens.append(tokendef) - return Terminal(Token('TERMINAL', token_name, -1), filter_out=isinstance(p, PatternStr)) + return Terminal(token_name, filter_out=isinstance(p, PatternStr)) def _rfind(s, choices): diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py index 57e4a4a..f3a5dd6 100644 --- a/lark/parsers/xearley.py +++ b/lark/parsers/xearley.py @@ -20,7 +20,7 @@ from collections import defaultdict -from ..exceptions import ParseError, UnexpectedInput +from ..exceptions import ParseError, UnexpectedCharacters from ..lexer import Token from ..tree import Tree from .grammar_analysis import GrammarAnalyzer @@ -116,7 +116,7 @@ class Parser: del delayed_matches[i+1] # No longer needed, so unburden memory if not next_set and not delayed_matches: - raise UnexpectedInput(stream, i, text_line, text_column, {item.expect for item in to_scan}, set(to_scan)) + raise UnexpectedCharacters(stream, i, text_line, text_column, {item.expect for item in to_scan}, set(to_scan)) return next_set