|
|
@@ -1,6 +1,7 @@ |
|
|
|
## Lexer Implementation |
|
|
|
|
|
|
|
import re |
|
|
|
import sre_parse |
|
|
|
|
|
|
|
from .utils import Str, classify |
|
|
|
from .common import is_terminal, PatternStr, PatternRE, TokenDef |
|
|
@@ -89,6 +90,10 @@ class Lexer(object): |
|
|
|
except: |
|
|
|
raise LexError("Cannot compile token: %s: %s" % (t.name, t.pattern)) |
|
|
|
|
|
|
|
width = sre_parse.parse(t.pattern.to_regexp()).getwidth() |
|
|
|
if width[0] == 0: |
|
|
|
raise LexError("Lexer does not allow zero-width tokens. (%s: %s)" % (t.name, t.pattern)) |
|
|
|
|
|
|
|
token_names = {t.name for t in tokens} |
|
|
|
for t in ignore: |
|
|
|
if t not in token_names: |
|
|
|