From 96ebe949630c67d4f212cf19b2ac9c6ff9a6e0a0 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Wed, 1 Mar 2017 18:18:55 +0200 Subject: [PATCH] Solved issue #3: infinite loop due to zero-length tokens --- lark/lexer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lark/lexer.py b/lark/lexer.py index 349f999..5da1f89 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -1,6 +1,7 @@ ## Lexer Implementation import re +import sre_parse from .utils import Str, classify from .common import is_terminal, PatternStr, PatternRE, TokenDef @@ -89,6 +90,10 @@ class Lexer(object): except: raise LexError("Cannot compile token: %s: %s" % (t.name, t.pattern)) + width = sre_parse.parse(t.pattern.to_regexp()).getwidth() + if width[0] == 0: + raise LexError("Lexer does not allow zero-width tokens. (%s: %s)" % (t.name, t.pattern)) + token_names = {t.name for t in tokens} for t in ignore: if t not in token_names: