Solved issue #3: infinite loop due to zero-length tokens

lark/lexer.py

@@ -1,6 +1,7 @@
 ## Lexer Implementation
 
 import re
+import sre_parse
 
 from .utils import Str, classify
 from .common import is_terminal, PatternStr, PatternRE, TokenDef
@@ -89,6 +90,10 @@ class Lexer(object):
             except:
                 raise LexError("Cannot compile token: %s: %s" % (t.name, t.pattern))
 
+            width = sre_parse.parse(t.pattern.to_regexp()).getwidth()
+            if width[0] == 0:
+                raise LexError("Lexer does not allow zero-width tokens. (%s: %s)" % (t.name, t.pattern))
+
         token_names = {t.name for t in tokens}
         for t in ignore:
             if t not in token_names: