Fixes for new standalone (Issue #349)

lark/lark.py

@@ -250,6 +250,7 @@ class Lark(Serialize):
         options['postlex'] = postlex
         inst.options = LarkOptions.deserialize(options, memo)
         inst.rules = [Rule.deserialize(r, memo) for r in data['rules']]
+        inst.source = '<deserialized>'
         inst._prepare_callbacks()
         inst.parser = inst.parser_class.deserialize(data['parser'], memo, inst._callbacks, inst.options.postlex)
         return inst
@@ -290,4 +291,4 @@ class Lark(Serialize):
         "Parse the given text, according to the options provided. Returns a tree, unless specified otherwise."
         return self.parser.parse(text)
 
-###}
+###}

lark/lexer.py

@@ -278,8 +278,8 @@ class TraditionalLexer(Lexer):
     __serialize_namespace__ = TerminalDef,
 
     def _deserialize(self):
-        self.mres = build_mres(self.terminals)
-        self.callback = {}  # TODO implement
+        self.user_callbacks = {} # TODO implement
+        self.build()
 
 
     def __init__(self, terminals, ignore=(), user_callbacks={}):
@@ -304,19 +304,21 @@ class TraditionalLexer(Lexer):
         self.ignore_types = list(ignore)
 
         terminals.sort(key=lambda x:(-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name))
+        self.terminals = terminals
+        self.user_callbacks = user_callbacks
+        self.build()
 
-        terminals, self.callback = _create_unless(terminals)
+    def build(self):
+        terminals, self.callback = _create_unless(self.terminals)
         assert all(self.callback.values())
 
-        for type_, f in user_callbacks.items():
+        for type_, f in self.user_callbacks.items():
             if type_ in self.callback:
                 # Already a callback there, probably UnlessCallback
                 self.callback[type_] = CallChain(self.callback[type_], f, lambda t: t.type == type_)
             else:
                 self.callback[type_] = f
 
-        self.terminals = terminals
-
         self.mres = build_mres(terminals)
 
 
@@ -364,4 +366,4 @@ class ContextualLexer(Lexer):
             l.lexer = self.lexers[self.parser_state]
             l.state = self.parser_state
 
-###}
+###}

lark/utils.py

@@ -18,16 +18,7 @@ def classify_bool(seq, pred):
 
     return true_elems, false_elems
 
-def classify(seq, key=None, value=None):
-    d = {}
-    for item in seq:
-        k = key(item) if (key is not None) else item
-        v = value(item) if (value is not None) else item
-        if k in d:
-            d[k].append(v)
-        else:
-            d[k] = [v]
-    return d
+
 
 def bfs(initial, expand):
     open_q = deque(list(initial))
@@ -58,6 +49,18 @@ def _serialize(value, memo):
     return value
 
 ###{standalone
+def classify(seq, key=None, value=None):
+    d = {}
+    for item in seq:
+        k = key(item) if (key is not None) else item
+        v = value(item) if (value is not None) else item
+        if k in d:
+            d[k].append(v)
+        else:
+            d[k] = [v]
+    return d
+
+
 def _deserialize(data, namespace, memo):
     if isinstance(data, dict):
         if '__type__' in data: # Object

tests/test_parser.py

@@ -1451,7 +1451,8 @@ def _make_parser_test(LEXER, PARSER):
         @unittest.skipIf(PARSER!='lalr', "Serialize currently only works for LALR parsers (though it should be easy to extend)")
         def test_serialize(self):
             grammar = """
-                start: "A" b "C"
+                start: _ANY b "C"
+                _ANY: /./
                 b: "B"
             """
             parser = _Lark(grammar)