Browse Source

BUGFIX: Create unless returned duplicate tokens (Issue #56)

tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.5.2
Erez Shinan 6 years ago
parent
commit
f63174a22c
1 changed files with 4 additions and 6 deletions
  1. +4
    -6
      lark/lexer.py

+ 4
- 6
lark/lexer.py View File

@@ -43,7 +43,7 @@ class Token(Str):
def __eq__(self, other):
if isinstance(other, Token) and self.type != other.type:
return False
return Str.__eq__(self, other)

__hash__ = Str.__hash__
@@ -74,7 +74,6 @@ def _create_unless(tokens):
tokens_by_type = classify(tokens, lambda t: type(t.pattern))
assert len(tokens_by_type) <= 2, tokens_by_type.keys()
embedded_strs = set()
delayed_strs = []
callback = {}
for retok in tokens_by_type.get(PatternRE, []):
unless = [] # {}
@@ -82,14 +81,13 @@ def _create_unless(tokens):
s = strtok.pattern.value
m = re.match(retok.pattern.to_regexp(), s)
if m and m.group(0) == s:
if strtok.pattern.flags:
delayed_strs.append(strtok)
embedded_strs.add(strtok.name)
unless.append(strtok)
if strtok.pattern.flags <= retok.pattern.flags:
embedded_strs.add(strtok)
if unless:
callback[retok.name] = _create_unless_callback(unless)

tokens = [t for t in tokens if t.name not in embedded_strs] + delayed_strs
tokens = [t for t in tokens if t not in embedded_strs]
return tokens, callback




Loading…
Cancel
Save