diff --git a/lark/lark.py b/lark/lark.py index 0d143df..f29d444 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -3,7 +3,7 @@ import sys, os, pickle, hashlib import tempfile from typing import ( TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, - Tuple, Iterable, IO, Any, TYPE_CHECKING + Tuple, Iterable, IO, Any, TYPE_CHECKING, Collection ) if TYPE_CHECKING: from .parsers.lalr_interactive_parser import InteractiveParser @@ -416,7 +416,7 @@ class Lark(Serialize): assert cache_md5 is not None f.write(cache_md5.encode('utf8') + b'\n') pickle.dump(used_files, f) - self.save(f) + self.save(f, _LOAD_ALLOWED_OPTIONS) if __doc__: __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC @@ -451,12 +451,14 @@ class Lark(Serialize): parser_conf = ParserConf(self.rules, self._callbacks, self.options.start) return parser_class(self.lexer_conf, parser_conf, options=self.options) - def save(self, f): + def save(self, f, exclude_options: Collection[str] = ()): """Saves the instance into the given file object Useful for caching and multiprocessing. """ data, m = self.memo_serialize([TerminalDef, Rule]) + if exclude_options: + data["options"] = {n: v for n, v in data["options"].items() if n not in exclude_options} pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL) @classmethod diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index 06533e0..d0703f2 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -42,7 +42,7 @@ class MakeParsingFrontend: class ParsingFrontend(Serialize): - __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser', 'options' + __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser' def __init__(self, lexer_conf, parser_conf, options, parser=None): self.parser_conf = parser_conf diff --git a/tests/test_cache.py b/tests/test_cache.py index 9f71552..5778632 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -52,6 +52,9 @@ class InlineTestT(Transformer): def NUM(self, token): return int(token) + def __reduce__(self): + raise TypeError("This Transformer should not be pickled.") + def append_zero(t): return t.update(value=t.value + '0') @@ -107,6 +110,8 @@ class TestCache(TestCase): def test_inline(self): # Test inline transformer (tree-less) & lexer_callbacks + # Note: the Transformer should not be saved to the file, + # and is made unpickable to check for that g = """ start: add+ add: NUM "+" NUM @@ -134,7 +139,7 @@ class TestCache(TestCase): assert len(self.mock_fs.files) == 1 res = parser.parse("ab") self.assertEqual(res, Tree('startab', [Tree('expr', ['a', 'b'])])) - + diff --git a/tests/test_nearley/nearley b/tests/test_nearley/nearley index 3268316..a46b374 160000 --- a/tests/test_nearley/nearley +++ b/tests/test_nearley/nearley @@ -1 +1 @@ -Subproject commit 326831689826cb1b9a4d21d1ce0d5db9278e9636 +Subproject commit a46b37471db486db0f6e1ce6a2934fb238346b44