From a787ea83d7a51960614450a7bb5fa551a5f2a5a9 Mon Sep 17 00:00:00 2001 From: MegaIng Date: Sat, 18 Sep 2021 22:13:37 +0200 Subject: [PATCH] Prevents caching values that are to be provided again next time anyway. (Most notably transformer, since those might not be picklable) --- lark/lark.py | 8 +++++--- lark/parser_frontends.py | 2 +- tests/test_nearley/nearley | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lark/lark.py b/lark/lark.py index 0d143df..f29d444 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -3,7 +3,7 @@ import sys, os, pickle, hashlib import tempfile from typing import ( TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, - Tuple, Iterable, IO, Any, TYPE_CHECKING + Tuple, Iterable, IO, Any, TYPE_CHECKING, Collection ) if TYPE_CHECKING: from .parsers.lalr_interactive_parser import InteractiveParser @@ -416,7 +416,7 @@ class Lark(Serialize): assert cache_md5 is not None f.write(cache_md5.encode('utf8') + b'\n') pickle.dump(used_files, f) - self.save(f) + self.save(f, _LOAD_ALLOWED_OPTIONS) if __doc__: __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC @@ -451,12 +451,14 @@ class Lark(Serialize): parser_conf = ParserConf(self.rules, self._callbacks, self.options.start) return parser_class(self.lexer_conf, parser_conf, options=self.options) - def save(self, f): + def save(self, f, exclude_options: Collection[str] = ()): """Saves the instance into the given file object Useful for caching and multiprocessing. """ data, m = self.memo_serialize([TerminalDef, Rule]) + if exclude_options: + data["options"] = {n: v for n, v in data["options"].items() if n not in exclude_options} pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL) @classmethod diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index 06533e0..d0703f2 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -42,7 +42,7 @@ class MakeParsingFrontend: class ParsingFrontend(Serialize): - __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser', 'options' + __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser' def __init__(self, lexer_conf, parser_conf, options, parser=None): self.parser_conf = parser_conf diff --git a/tests/test_nearley/nearley b/tests/test_nearley/nearley index 3268316..a46b374 160000 --- a/tests/test_nearley/nearley +++ b/tests/test_nearley/nearley @@ -1 +1 @@ -Subproject commit 326831689826cb1b9a4d21d1ce0d5db9278e9636 +Subproject commit a46b37471db486db0f6e1ce6a2934fb238346b44