diff --git a/lark/common.py b/lark/common.py index 06e1a38..76155f3 100644 --- a/lark/common.py +++ b/lark/common.py @@ -1,8 +1,4 @@ -###{standalone -###} - - class LexerConf: def __init__(self, tokens, ignore=(), postlex=None, callbacks=None): diff --git a/lark/lexer.py b/lark/lexer.py index bbf1053..64d18d1 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -284,6 +284,7 @@ class TraditionalLexer(Lexer): self.mres = build_mres(terminals) + def lex(self, stream): return _Lex(self).lex(stream, self.newline_types, self.ignore_types) diff --git a/lark/tools/standalone.py b/lark/tools/standalone.py index 1d35222..51f1ec3 100644 --- a/lark/tools/standalone.py +++ b/lark/tools/standalone.py @@ -54,7 +54,6 @@ EXTRACT_STANDALONE_FILES = [ 'tools/standalone.py', 'exceptions.py', 'utils.py', - 'common.py', 'tree.py', 'visitors.py', 'indenter.py', @@ -83,12 +82,15 @@ def extract_sections(lines): return {name:''.join(text) for name, text in sections.items()} +def _prepare_mres(mres): + return [(p.pattern,{i: t for i, t in d.items()}) for p,d in mres] + class TraditionalLexerAtoms: def __init__(self, lexer): - self.mres = [(p.pattern,d) for p,d in lexer.mres] + self.mres = _prepare_mres(lexer.mres) self.newline_types = lexer.newline_types self.ignore_types = lexer.ignore_types - self.callback = {name:[(p.pattern,d) for p,d in c.mres] + self.callback = {name:_prepare_mres(c.mres) for name, c in lexer.callback.items()} def print_python(self):