Merge pull request #1003 from MegaIng/fix-cache

Fix loading from cache

lark/lark.py

@@ -3,7 +3,7 @@ import sys, os, pickle, hashlib
 import tempfile
 from typing import (
     TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional,
-    Tuple, Iterable, IO, Any, TYPE_CHECKING
+    Tuple, Iterable, IO, Any, TYPE_CHECKING, Collection
 )
 if TYPE_CHECKING:
     from .parsers.lalr_interactive_parser import InteractiveParser
@@ -416,7 +416,7 @@ class Lark(Serialize):
                 assert cache_md5 is not None
                 f.write(cache_md5.encode('utf8') + b'\n')
                 pickle.dump(used_files, f)
-                self.save(f)
+                self.save(f, _LOAD_ALLOWED_OPTIONS)
 
     if __doc__:
         __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC
@@ -451,12 +451,14 @@ class Lark(Serialize):
         parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
         return parser_class(self.lexer_conf, parser_conf, options=self.options)
 
-    def save(self, f):
+    def save(self, f, exclude_options: Collection[str] = ()):
         """Saves the instance into the given file object
 
         Useful for caching and multiprocessing.
         """
         data, m = self.memo_serialize([TerminalDef, Rule])
+        if exclude_options:
+            data["options"] = {n: v for n, v in data["options"].items() if n not in exclude_options}
         pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL)
 
     @classmethod

lark/parser_frontends.py

@@ -42,7 +42,7 @@ class MakeParsingFrontend:
 
 
 class ParsingFrontend(Serialize):
-    __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser', 'options'
+    __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser'
 
     def __init__(self, lexer_conf, parser_conf, options, parser=None):
         self.parser_conf = parser_conf

tests/test_cache.py

@@ -52,6 +52,9 @@ class InlineTestT(Transformer):
     def NUM(self, token):
         return int(token)
 
+    def __reduce__(self):
+        raise TypeError("This Transformer should not be pickled.")
+
 
 def append_zero(t):
     return t.update(value=t.value + '0')
@@ -107,6 +110,8 @@ class TestCache(TestCase):
     
     def test_inline(self):
         # Test inline transformer (tree-less) & lexer_callbacks
+        # Note: the Transformer should not be saved to the file,
+        #       and is made unpickable to check for that
         g = """
         start: add+
         add: NUM "+" NUM
@@ -134,7 +139,7 @@ class TestCache(TestCase):
         assert len(self.mock_fs.files) == 1
         res = parser.parse("ab")
         self.assertEqual(res, Tree('startab', [Tree('expr', ['a', 'b'])]))
-        
+
         
 
 

tests/test_nearley/nearley

@@ -1 +1 @@
-Subproject commit 326831689826cb1b9a4d21d1ce0d5db9278e9636
+Subproject commit a46b37471db486db0f6e1ce6a2934fb238346b44