Browse Source

Merge branch 'fix-debug-cache-incompatibility' of https://github.com/ldbo/lark into ldbo-fix-debug-cache-incompatibility

tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.11.0
Erez Sh 4 years ago
parent
commit
e5cf495230
5 changed files with 17 additions and 10 deletions
  1. +4
    -2
      lark/lark.py
  2. +4
    -2
      lark/parser_frontends.py
  3. +2
    -2
      lark/parsers/lalr_parser.py
  4. +6
    -3
      tests/test_cache.py
  5. +1
    -1
      tests/test_trees.py

+ 4
- 2
lark/lark.py View File

@@ -336,7 +336,8 @@ class Lark(Serialize):
Useful for caching and multiprocessing.
"""
data, m = self.memo_serialize([TerminalDef, Rule])
pickle.dump({'data': data, 'memo': m}, f)
pickle.dump({'data': data, 'memo': m}, f,
protocol=pickle.HIGHEST_PROTOCOL)

@classmethod
def load(cls, f):
@@ -373,7 +374,8 @@ class Lark(Serialize):
self._callbacks,
self.options.postlex,
self.options.transformer,
re_module
re_module,
self.options.debug
)
self.terminals = self.parser.lexer_conf.tokens
self._terminals_dict = {t.name: t for t in self.terminals}


+ 4
- 2
lark/parser_frontends.py View File

@@ -82,11 +82,13 @@ class WithLexer(_ParserFrontend):
self.postlex = lexer_conf.postlex

@classmethod
def deserialize(cls, data, memo, callbacks, postlex, transformer, re_module):
def deserialize(cls, data, memo, callbacks, postlex, transformer, re_module,
debug):
inst = super(WithLexer, cls).deserialize(data, memo)

inst.postlex = postlex
inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks)
inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks,
debug)

terminals = [item for item in memo.values() if isinstance(item, TerminalDef)]
inst.lexer_conf.callbacks = _get_lexer_callbacks(transformer, terminals)


+ 2
- 2
lark/parsers/lalr_parser.py View File

@@ -23,10 +23,10 @@ class LALR_Parser(object):
self.parser = _Parser(analysis.parse_table, callbacks, debug)

@classmethod
def deserialize(cls, data, memo, callbacks):
def deserialize(cls, data, memo, callbacks, debug=False):
inst = cls.__new__(cls)
inst._parse_table = IntParseTable.deserialize(data, memo)
inst.parser = _Parser(inst._parse_table, callbacks)
inst.parser = _Parser(inst._parse_table, callbacks, debug)
return inst

def serialize(self, memo):


+ 6
- 3
tests/test_cache.py View File

@@ -86,6 +86,12 @@ class TestCache(TestCase):
parser = Lark(g, parser='lalr', lexer=CustomLexer, cache=True)
assert len(mock_fs.files) == 1
assert parser.parse('a') == Tree('start', [])

# Test options persistence
mock_fs.files = {}
Lark(g, parser="lalr", debug=True, cache=True)
parser = Lark(g, parser="lalr", debug=True, cache=True)
assert parser.options.options['debug']
finally:
lark_module.FS = fs

@@ -93,6 +99,3 @@ class TestCache(TestCase):

if __name__ == '__main__':
main()




+ 1
- 1
tests/test_trees.py View File

@@ -20,7 +20,7 @@ class TestTrees(TestCase):

def test_pickle(self):
s = copy.deepcopy(self.tree1)
data = pickle.dumps(s)
data = pickle.dumps(s, protocol=pickle.HIGHEST_PROTOCOL)
assert pickle.loads(data) == s

def test_repr_runnable(self):


Loading…
Cancel
Save