Browse Source

BUGIX in lexer: Embedding strings overwrote priority (Issue #121)

tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.6.0
Erez Shinan 6 years ago
parent
commit
0f0776c0fa
2 changed files with 14 additions and 0 deletions
  1. +2
    -0
      lark/lexer.py
  2. +12
    -0
      tests/test_parser.py

+ 2
- 0
lark/lexer.py View File

@@ -143,6 +143,8 @@ def _create_unless(tokens):
for retok in tokens_by_type.get(PatternRE, []):
unless = [] # {}
for strtok in tokens_by_type.get(PatternStr, []):
if strtok.priority > retok.priority:
continue
s = strtok.pattern.value
m = re.match(retok.pattern.to_regexp(), s)
if m and m.group(0) == s:


+ 12
- 0
tests/test_parser.py View File

@@ -1173,6 +1173,18 @@ def _make_parser_test(LEXER, PARSER):
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB')
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAABB')

@unittest.skipIf(PARSER=='earley', "Priority not handled correctly right now") # TODO XXX
def test_priority_vs_embedded(self):
g = """
A.2: "a"
WORD: ("a".."z")+

start: (A | WORD)+
"""
l = _Lark(g)
t = l.parse('abc')
self.assertEqual(t.children, ['a', 'bc'])
self.assertEqual(t.children[0].type, 'A')





Loading…
Cancel
Save