Added more information in UnexpectedInput exception (Issue #78)

lark/lexer.py

@@ -13,6 +13,8 @@ class UnexpectedInput(LexError):
     def __init__(self, seq, lex_pos, line, column, allowed=None):
         context = seq[lex_pos:lex_pos+5]
         message = "No token defined for: '%s' in %r at line %d col %d" % (seq[lex_pos], context, line, column)
+        if allowed:
+            message += '\n\nExpecting: %s\n' % allowed
 
         super(UnexpectedInput, self).__init__(message)
 

lark/load_grammar.py

@@ -347,7 +347,7 @@ class PrepareLiterals(InlineTransformer):
         assert start.type == end.type == 'STRING'
         start = start.value[1:-1]
         end = end.value[1:-1]
-        assert len(start) == len(end) == 1
+        assert len(start) == len(end) == 1, (start, end, len(start), len(end))
         regexp = '[%s-%s]' % (start, end)
         return T('pattern', [PatternRE(regexp)])
 

lark/parsers/xearley.py

@@ -112,7 +112,7 @@ class Parser:
             del delayed_matches[i+1]    # No longer needed, so unburden memory
 
             if not next_set and not delayed_matches:
-                raise UnexpectedInput(stream, i, text_line, text_column, to_scan)
+                raise UnexpectedInput(stream, i, text_line, text_column, {item.expect for item in to_scan})
 
             return next_set