@@ -27,7 +27,7 @@ class WithLexer(Serialize): | |||||
return inst | return inst | ||||
def _serialize(self, data, memo): | def _serialize(self, data, memo): | ||||
data['parser'] = data['parser'].serialize() | |||||
data['parser'] = data['parser'].serialize(memo) | |||||
def init_traditional_lexer(self, lexer_conf): | def init_traditional_lexer(self, lexer_conf): | ||||
self.lexer_conf = lexer_conf | self.lexer_conf = lexer_conf | ||||
@@ -13,6 +13,7 @@ from ..utils import classify, classify_bool, bfs, fzset, Serialize, Enumerator | |||||
from ..exceptions import GrammarError | from ..exceptions import GrammarError | ||||
from .grammar_analysis import GrammarAnalyzer, Terminal | from .grammar_analysis import GrammarAnalyzer, Terminal | ||||
from ..grammar import Rule | |||||
###{standalone | ###{standalone | ||||
@@ -33,19 +34,18 @@ class ParseTable: | |||||
self.start_state = start_state | self.start_state = start_state | ||||
self.end_state = end_state | self.end_state = end_state | ||||
def serialize(self): | |||||
def serialize(self, memo): | |||||
tokens = Enumerator() | tokens = Enumerator() | ||||
rules = Enumerator() | rules = Enumerator() | ||||
states = { | states = { | ||||
state: {tokens.get(token): ((1, rules.get(arg)) if action is Reduce else (0, arg)) | |||||
state: {tokens.get(token): ((1, arg.serialize(memo)) if action is Reduce else (0, arg)) | |||||
for token, (action, arg) in actions.items()} | for token, (action, arg) in actions.items()} | ||||
for state, actions in self.states.items() | for state, actions in self.states.items() | ||||
} | } | ||||
return { | return { | ||||
'tokens': tokens.reversed(), | 'tokens': tokens.reversed(), | ||||
'rules': {idx: r.serialize() for idx, r in rules.reversed().items()}, | |||||
'states': states, | 'states': states, | ||||
'start_state': self.start_state, | 'start_state': self.start_state, | ||||
'end_state': self.end_state, | 'end_state': self.end_state, | ||||
@@ -54,9 +54,8 @@ class ParseTable: | |||||
@classmethod | @classmethod | ||||
def deserialize(cls, data, memo): | def deserialize(cls, data, memo): | ||||
tokens = data['tokens'] | tokens = data['tokens'] | ||||
rules = data['rules'] | |||||
states = { | states = { | ||||
state: {tokens[token]: ((Reduce, rules[arg]) if action==1 else (Shift, arg)) | |||||
state: {tokens[token]: ((Reduce, Rule.deserialize(arg, memo)) if action==1 else (Shift, arg)) | |||||
for token, (action, arg) in actions.items()} | for token, (action, arg) in actions.items()} | ||||
for state, actions in data['states'].items() | for state, actions in data['states'].items() | ||||
} | } | ||||
@@ -28,8 +28,8 @@ class LALR_Parser(object): | |||||
inst.parser = _Parser(IntParseTable.deserialize(data, memo), callbacks) | inst.parser = _Parser(IntParseTable.deserialize(data, memo), callbacks) | ||||
return inst | return inst | ||||
def serialize(self): | |||||
return self._parse_table.serialize() | |||||
def serialize(self, memo): | |||||
return self._parse_table.serialize(memo) | |||||
def parse(self, *args): | def parse(self, *args): | ||||
return self.parser.parse(*args) | return self.parser.parse(*args) | ||||
@@ -49,7 +49,7 @@ from lark import Lark | |||||
from lark.parsers.lalr_analysis import Reduce | from lark.parsers.lalr_analysis import Reduce | ||||
from lark.grammar import RuleOptions | |||||
from lark.grammar import RuleOptions, Rule | |||||
from lark.lexer import TerminalDef | from lark.lexer import TerminalDef | ||||
_dir = path.dirname(__file__) | _dir = path.dirname(__file__) | ||||
@@ -63,13 +63,13 @@ EXTRACT_STANDALONE_FILES = [ | |||||
'tree.py', | 'tree.py', | ||||
'visitors.py', | 'visitors.py', | ||||
'indenter.py', | 'indenter.py', | ||||
'grammar.py', | |||||
'lexer.py', | 'lexer.py', | ||||
'parse_tree_builder.py', | 'parse_tree_builder.py', | ||||
'parsers/lalr_parser.py', | 'parsers/lalr_parser.py', | ||||
'parsers/lalr_analysis.py', | 'parsers/lalr_analysis.py', | ||||
'parser_frontends.py', | 'parser_frontends.py', | ||||
'lark.py', | 'lark.py', | ||||
'grammar.py', | |||||
] | ] | ||||
def extract_sections(lines): | def extract_sections(lines): | ||||
@@ -101,7 +101,7 @@ def main(fobj, start): | |||||
with open(os.path.join(_larkdir, pyfile)) as f: | with open(os.path.join(_larkdir, pyfile)) as f: | ||||
print (extract_sections(f)['standalone']) | print (extract_sections(f)['standalone']) | ||||
data, m = lark_inst.memo_serialize([TerminalDef]) | |||||
data, m = lark_inst.memo_serialize([TerminalDef, Rule]) | |||||
print( 'DATA = (' ) | print( 'DATA = (' ) | ||||
# pprint(data, width=160) | # pprint(data, width=160) | ||||
print(data) | print(data) | ||||
@@ -113,10 +113,9 @@ def main(fobj, start): | |||||
print('Shift = 0') | print('Shift = 0') | ||||
print('Reduce = 1') | print('Reduce = 1') | ||||
print("def load_parser():") | |||||
print(" return Lark.deserialize(DATA)") | |||||
print("def Lark_StandAlone():") | |||||
print(" memo = SerializeMemoizer.deserialize(MEMO, {'Rule': Rule, 'TerminalDef': TerminalDef}, {})") | |||||
print(" return Lark.deserialize(DATA, memo)") | |||||
@@ -42,14 +42,11 @@ def bfs(initial, expand): | |||||
###{standalone | |||||
import sys, re | |||||
Py36 = (sys.version_info[:2] >= (3, 6)) | |||||
def _serialize(value, memo): | def _serialize(value, memo): | ||||
# if memo and memo.in_types(value): | |||||
# return {'__memo__': memo.memoized.get(value)} | |||||
if isinstance(value, Serialize): | if isinstance(value, Serialize): | ||||
return value.serialize(memo) | return value.serialize(memo) | ||||
elif isinstance(value, list): | elif isinstance(value, list): | ||||
@@ -60,11 +57,14 @@ def _serialize(value, memo): | |||||
return {key:_serialize(elem, memo) for key, elem in value.items()} | return {key:_serialize(elem, memo) for key, elem in value.items()} | ||||
return value | return value | ||||
###{standalone | |||||
def _deserialize(data, namespace, memo): | def _deserialize(data, namespace, memo): | ||||
if isinstance(data, dict): | if isinstance(data, dict): | ||||
if '__type__' in data: # Object | if '__type__' in data: # Object | ||||
class_ = namespace[data['__type__']] | class_ = namespace[data['__type__']] | ||||
return class_.deserialize(data, memo) | return class_.deserialize(data, memo) | ||||
elif '__memo__' in data: | |||||
return memo[data['__memo__']] | |||||
return {key:_deserialize(value, namespace, memo) for key, value in data.items()} | return {key:_deserialize(value, namespace, memo) for key, value in data.items()} | ||||
elif isinstance(data, list): | elif isinstance(data, list): | ||||
return [_deserialize(value, namespace, memo) for value in data] | return [_deserialize(value, namespace, memo) for value in data] | ||||
@@ -159,6 +159,11 @@ def smart_decorator(f, create_decorator): | |||||
else: | else: | ||||
return create_decorator(f.__func__.__call__, True) | return create_decorator(f.__func__.__call__, True) | ||||
import sys, re | |||||
Py36 = (sys.version_info[:2] >= (3, 6)) | |||||
###} | |||||
def dedup_list(l): | def dedup_list(l): | ||||
"""Given a list (l) will removing duplicates from the list, | """Given a list (l) will removing duplicates from the list, | ||||
preserving the original order of the list. Assumes that | preserving the original order of the list. Assumes that | ||||
@@ -166,7 +171,7 @@ def dedup_list(l): | |||||
dedup = set() | dedup = set() | ||||
return [ x for x in l if not (x in dedup or dedup.add(x))] | return [ x for x in l if not (x in dedup or dedup.add(x))] | ||||
###} | |||||
try: | try: | ||||