Browse Source

Better error reporting in lexer and lalr parser

tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.5.1
Erez Shinan 7 years ago
parent
commit
2e8f74c774
4 changed files with 10 additions and 4 deletions
  1. +2
    -2
      lark/lexer.py
  2. +1
    -0
      lark/load_grammar.py
  3. +5
    -2
      lark/parser.py
  4. +2
    -0
      lark/tests/test_parser.py

+ 2
- 2
lark/lexer.py View File

@@ -61,7 +61,7 @@ class Lexer(object):

def lex(self, stream):
lex_pos = 0
line = 0
line = 1
col_start_pos = 0
while True:
i = 0
@@ -87,7 +87,7 @@ class Lexer(object):
else:
if lex_pos < len(stream):
context = stream[lex_pos:lex_pos+5]
raise LexError("No token defined for: '%s' in %s" % (stream[lex_pos], context))
raise LexError("No token defined for: '%s' in %s at line %d" % (stream[lex_pos], context, line))
break



+ 1
- 0
lark/load_grammar.py View File

@@ -231,6 +231,7 @@ class ExtractAnonTokens(InlineTransformer):
self.tokens = tokens
self.token_set = token_set
self.token_reverse = {value[1:-1]: name for name, value, _flags in tokens}
self.i = 0

def anontoken(self, token):
if token.type == 'STRING':


+ 5
- 2
lark/parser.py View File

@@ -22,8 +22,11 @@ class Parser(object):
return states_idx[state][key]
except KeyError:
expected = states_idx[state].keys()
context = ' '.join(['%s(%r)' % (t.type, t.value) for t in seq[i:i+5]])
raise ParseError("Unexpected input %r.\nExpected: %s\nContext: %s" % (key, expected, context))
context = ' '.join(['%r(%s)' % (t.value, t.type) for t in seq[i:i+5]])
token = seq[i]
raise ParseError("Unexpected input %r at line %d, column %d.\n"
"Expected: %s\n"
"Context: %s" % (token.value, token.line, token.column, expected, context))

def reduce(rule):
if rule.expansion:


+ 2
- 0
lark/tests/test_parser.py View File

@@ -267,6 +267,8 @@ class TestLalr(unittest.TestCase):
""", parser='lalr')
x = g.parse('Hello World')
self.assertSequenceEqual(x.children, ['World'])
x = g.parse('HelloWorld')
self.assertSequenceEqual(x.children, ['HelloWorld'])

def test_undefined_rule(self):
self.assertRaises(GrammarError, Lark, """start: a""", parser='lalr')


Loading…
Cancel
Save