diff --git a/lark/tools/nearley.py b/lark/tools/nearley.py index 5613c21..9b14db6 100644 --- a/lark/tools/nearley.py +++ b/lark/tools/nearley.py @@ -3,18 +3,19 @@ import os.path import sys -from lark import Lark, InlineTransformer + +from lark import Lark, InlineTransformer, Transformer nearley_grammar = r""" start: (ruledef|directive)+ directive: "@" NAME (STRING|NAME) - | "@" _JS -> js_code + | "@" JS -> js_code ruledef: NAME "->" expansions | NAME REGEXP "->" expansions -> macro expansions: expansion ("|" expansion)* - expansion: expr+ _JS? + expansion: expr+ js ?expr: item [":" /[+*?]/] @@ -24,7 +25,8 @@ nearley_grammar = r""" rule: NAME string: STRING regexp: REGEXP - _JS: /(?s){%.*?%}/ + JS: /(?s){%.*?%}/ + js: JS? NAME: /[a-zA-Z_$]\w*/ COMMENT: /\#[^\n]*/ @@ -37,61 +39,129 @@ nearley_grammar = r""" """ +nearley_grammar_parser = Lark(nearley_grammar, parser='earley', lexer='standard') +def _get_rulename(name): + name = {'_': '_ws_maybe', '__':'_ws'}.get(name, name) + return 'n_' + name.replace('$', '__DOLLAR__').lower() class NearleyToLark(InlineTransformer): - def __init__(self, builtin_path): - self.builtin_path = builtin_path + def __init__(self): + self._count = 0 + self.extra_rules = {} + self.extra_rules_rev = {} + self.alias_js_code = {} + + def _new_function(self, code): + name = 'alias_%d' % self._count + self._count += 1 + + self.alias_js_code[name] = code + return name + + def _extra_rule(self, rule): + if rule in self.extra_rules_rev: + return self.extra_rules_rev[rule] + + name = 'xrule_%d' % len(self.extra_rules) + assert name not in self.extra_rules + self.extra_rules[name] = rule + self.extra_rules_rev[rule] = name + return name def rule(self, name): - # return {'_': '_WS?', '__':'_WS'}.get(name, name) - return {'_': '_ws_maybe', '__':'_ws'}.get(name, name) + return _get_rulename(name) def ruledef(self, name, exps): - name = {'_': '_ws_maybe', '__':'_ws'}.get(name, name) - return '%s: %s' % (name, exps) + return '!%s: %s' % (_get_rulename(name), exps) def expr(self, item, op): - return '(%s)%s' % (item, op) + rule = '(%s)%s' % (item, op) + return self._extra_rule(rule) def regexp(self, r): return '/%s/' % r def string(self, s): - # TODO allow regular strings, and split them in the parser frontend - return ' '.join('"%s"'%ch for ch in s[1:-1]) + return self._extra_rule(s) def expansion(self, *x): - return ' '.join(x) + x, js = x[:-1], x[-1] + if js.children: + js_code ,= js.children + js_code = js_code[2:-2] + alias = '-> ' + self._new_function(js_code) + else: + alias = '' + return ' '.join(x) + alias def expansions(self, *x): - return '(%s)' % ('\n |'.join(x)) - - def js_code(self): - return '' - - def macro(self, *args): - return '' # TODO support macros?! - - def directive(self, name, *args): - if name == 'builtin': - arg = args[0][1:-1] - with open(os.path.join(self.builtin_path, arg)) as f: - text = f.read() - return nearley_to_lark(text, self.builtin_path) - elif name == 'preprocessor': - return '' - - raise Exception('Unknown directive: %s' % name) + return '%s' % ('\n |'.join(x)) def start(self, *rules): return '\n'.join(filter(None, rules)) -def nearley_to_lark(g, builtin_path): - parser = Lark(nearley_grammar, parser='earley', lexer='standard') - tree = parser.parse(g) - return NearleyToLark(builtin_path).transform(tree) - +def _nearley_to_lark(g, builtin_path, n2l, js_code): + rule_defs = [] + + tree = nearley_grammar_parser.parse(g) + for statement in tree.children: + if statement.data == 'directive': + directive, arg = statement.children + if directive == 'builtin': + with open(os.path.join(builtin_path, arg[1:-1])) as f: + text = f.read() + rule_defs += _nearley_to_lark(text, builtin_path, n2l, js_code) + else: + assert False, directive + elif statement.data == 'js_code': + code ,= statement.children + code = code[2:-2] + js_code.append(code) + elif statement.data == 'macro': + pass # TODO Add support for macros! + elif statement.data == 'ruledef': + rule_defs.append( n2l.transform(statement) ) + else: + raise Exception("Unknown statement: %s" % statement) + + return rule_defs + + +def create_code_for_nearley_grammar(g, start, builtin_path): + import js2py + + emit_code = [] + def emit(x=None): + if x: + emit_code.append(x) + emit_code.append('\n') + + js_code = ['function id(x) {return x[0];}'] + n2l = NearleyToLark() + lark_g = '\n'.join(_nearley_to_lark(g, builtin_path, n2l, js_code)) + lark_g += '\n'+'\n'.join('!%s: %s' % item for item in n2l.extra_rules.items()) + + emit('from lark import Lark, Transformer') + emit() + emit('grammar = ' + repr(lark_g)) + emit() + + for alias, code in n2l.alias_js_code.items(): + js_code.append('%s = (%s);' % (alias, code)) + + emit(js2py.translate_js('\n'.join(js_code))) + emit('class TranformNearley(Transformer):') + for alias in n2l.alias_js_code: + emit(" %s = var.get('%s').to_python()" % (alias, alias)) + emit(" __default__ = lambda self, n, c: c if c else None") + + emit() + emit('parser = Lark(grammar, start="n_%s")' % start) + emit('def parse(text):') + emit(' return TranformNearley().transform(parser.parse(text))') + + return ''.join(emit_code) def test(): css_example_grammar = """ @@ -129,24 +199,25 @@ def test(): function(d) {return Math.floor(d[0]*255); } %} """ - converted_grammar = nearley_to_lark(css_example_grammar, '/home/erez/nearley/builtin') - print(converted_grammar) + code = create_code_for_nearley_grammar(css_example_grammar, 'csscolor', '/home/erez/nearley/builtin') + d = {} + exec (code, d) + parse = d['parse'] - l = Lark(converted_grammar, start='csscolor') - print(l.parse('#a199ff').pretty()) - print(l.parse('rgb(255, 70%, 3)').pretty()) + print(parse('#a199ff')) + print(parse('rgb(255, 70%, 3)')) def main(): - try: - nearley_lib = sys.argv[1] - except IndexError: - print("Reads Nearley grammar from stdin and outputs a lark grammar.") - print("Usage: %s " % sys.argv[0]) + if len(sys.argv) < 3: + print("Reads Nearley grammar (with js functions) outputs an equivalent lark parser.") + print("Usage: %s " % sys.argv[0]) return - grammar = sys.stdin.read() - print(nearley_to_lark(grammar, os.path.join(nearley_lib, 'builtin'))) + fn, start, nearley_lib = sys.argv[1:] + with open(fn) as f: + grammar = f.read() + print(create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'))) if __name__ == '__main__':