From b951b5f79cf1ac79ad70e44d7038bf214ece7d2b Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sat, 7 Apr 2018 14:55:17 +0300 Subject: [PATCH] Significant reduction in memory consumption (Saving only parse-table instead of analysis instance) --- lark/parser_frontends.py | 4 ++-- lark/parsers/earley.py | 6 +++--- lark/parsers/lalr_parser.py | 3 ++- lark/tools/standalone.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index c4a6f9b..a36252c 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -15,9 +15,9 @@ class WithLexer: def init_contextual_lexer(self, lexer_conf, parser_conf): self.lexer_conf = lexer_conf - d = {idx:t.keys() for idx, t in self.parser.analysis.parse_table.states.items()} + states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()} always_accept = lexer_conf.postlex.always_accept if lexer_conf.postlex else () - self.lexer = ContextualLexer(lexer_conf.tokens, d, ignore=lexer_conf.ignore, always_accept=always_accept, user_callbacks=lexer_conf.callbacks) + self.lexer = ContextualLexer(lexer_conf.tokens, states, ignore=lexer_conf.ignore, always_accept=always_accept, user_callbacks=lexer_conf.callbacks) def lex(self, text): stream = self.lexer.lex(text) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 2b04a1e..d119e41 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -145,16 +145,16 @@ class Column: class Parser: def __init__(self, parser_conf, term_matcher, resolve_ambiguity=None): - self.analysis = GrammarAnalyzer(parser_conf) + analysis = GrammarAnalyzer(parser_conf) self.parser_conf = parser_conf self.resolve_ambiguity = resolve_ambiguity - self.FIRST = self.analysis.FIRST + self.FIRST = analysis.FIRST self.postprocess = {} self.predictions = {} for rule in parser_conf.rules: self.postprocess[rule] = rule.alias if callable(rule.alias) else getattr(parser_conf.callback, rule.alias) - self.predictions[rule.origin] = [x.rule for x in self.analysis.expand_rule(rule.origin)] + self.predictions[rule.origin] = [x.rule for x in analysis.expand_rule(rule.origin)] self.term_matcher = term_matcher diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py index f4e0942..a20db07 100644 --- a/lark/parsers/lalr_parser.py +++ b/lark/parsers/lalr_parser.py @@ -11,11 +11,12 @@ class Parser: def __init__(self, parser_conf): assert all(r.options is None or r.options.priority is None for r in parser_conf.rules), "LALR doesn't yet support prioritization" - self.analysis = analysis = LALR_Analyzer(parser_conf) + analysis = LALR_Analyzer(parser_conf) analysis.compute_lookahead() callbacks = {rule: getattr(parser_conf.callback, rule.alias or rule.origin, None) for rule in parser_conf.rules} + self._parse_table = analysis.parse_table self.parser_conf = parser_conf self.parser = _Parser(analysis.parse_table, callbacks) self.parse = self.parser.parse diff --git a/lark/tools/standalone.py b/lark/tools/standalone.py index cf29cee..61ce94e 100644 --- a/lark/tools/standalone.py +++ b/lark/tools/standalone.py @@ -126,7 +126,7 @@ def _get_token_type(token_type): class ParserAtoms: def __init__(self, parser): - self.parse_table = parser.analysis.parse_table + self.parse_table = parser._parse_table def print_python(self): print('class ParseTable: pass')