diff --git a/examples/advanced/python3.lark b/examples/advanced/python3.lark index 803a21b..f20f972 100644 --- a/examples/advanced/python3.lark +++ b/examples/advanced/python3.lark @@ -23,9 +23,11 @@ decorated: decorators (classdef | funcdef | async_funcdef) async_funcdef: "async" funcdef funcdef: "def" NAME "(" parameters? ")" ["->" test] ":" suite -parameters: paramvalue ("," paramvalue)* ["," "/"] ["," [starparams | kwparams]] +parameters: paramvalue ("," paramvalue)* ["," SLASH] ["," [starparams | kwparams]] | starparams | kwparams + +SLASH: "/" // Otherwise the it will completely disappear and it will be undisguisable in the result starparams: "*" typedparam? ("," paramvalue)* ["," kwparams] kwparams: "**" typedparam @@ -116,9 +118,10 @@ AWAIT: "await" | atom_expr "." NAME -> getattr | atom -?atom: "(" [yield_expr|testlist_comp] ")" -> tuple +?atom: "(" [yield_expr|tuplelist_comp] ")" -> tuple | "[" [testlist_comp] "]" -> list - | "{" [dictorsetmaker] "}" -> dict + | "{" [dict_comp] "}" -> dict + | "{" set_comp "}" -> set | NAME -> var | number | string+ | "(" test ")" @@ -127,13 +130,23 @@ AWAIT: "await" | "True" -> const_true | "False" -> const_false -?testlist_comp: (test|star_expr) [comp_for | ("," (test|star_expr))+ [","] | ","] -subscriptlist: subscript ("," subscript)* [","] +?testlist_comp: test | tuplelist_comp +tuplelist_comp: (test|star_expr) (comp_for | ("," (test|star_expr))+ [","] | ",") +?subscriptlist: subscript + | subscript (("," subscript)+ [","] | ",") -> subscript_tuple subscript: test | [test] ":" [test] [sliceop] sliceop: ":" [test] -exprlist: (expr|star_expr) ("," (expr|star_expr))* [","] -testlist: test ("," test)* [","] -dictorsetmaker: ( ((test ":" test | "**" expr) (comp_for | ("," (test ":" test | "**" expr))* [","])) | ((test | star_expr) (comp_for | ("," (test | star_expr))* [","])) ) +exprlist: (expr|star_expr) + | (expr|star_expr) (("," (expr|star_expr))+ [","]|",") -> exprlist_tuple +testlist: test | testlist_tuple +testlist_tuple: test (("," test)+ [","] | ",") +dict_comp: key_value comp_for + | (key_value | "**" expr) ("," (key_value | "**" expr))* [","] + +key_value: test ":" test + +set_comp: test comp_for + | (test|star_expr) ("," (test | star_expr))* [","] classdef: "class" NAME ["(" [arguments] ")"] ":" suite diff --git a/examples/advanced/reconstruct_python.py b/examples/advanced/reconstruct_python.py new file mode 100644 index 0000000..b3d8992 --- /dev/null +++ b/examples/advanced/reconstruct_python.py @@ -0,0 +1,58 @@ +from lark import Token +from lark.reconstruct import Reconstructor + +from python_parser import python_parser3 + + +test_python = open(__file__).read() + +def special(sym): + return Token('SPECIAL', sym.name) + +SPACE_AFTER = set(',+-*/~@<>="|:') +SPACE_BEFORE = (SPACE_AFTER - set(',:')) | set('\'') + +def postproc(items): + stack = ['\n'] + actions = [] + last_was_whitespace = True + for item in items: + if isinstance(item, Token) and item.type == 'SPECIAL': + actions.append(item.value) + else: + if actions: + assert actions[0] == '_NEWLINE' and '_NEWLINE' not in actions[1:], actions + + for a in actions[1:]: + if a == '_INDENT': + stack.append(stack[-1] + ' ' * 4) + else: + assert a == '_DEDENT' + stack.pop() + actions.clear() + yield stack[-1] + last_was_whitespace = True + if not last_was_whitespace: + if item[0] in SPACE_BEFORE: + yield ' ' + yield item + last_was_whitespace = item[-1].isspace() + if not last_was_whitespace: + if item[-1] in SPACE_AFTER: + yield ' ' + last_was_whitespace = True + yield "\n" + + +tree = python_parser3.parse(test_python) + + +python_reconstruct = Reconstructor(python_parser3, {'_NEWLINE': special, '_DEDENT': special, '_INDENT': special}) + +output = python_reconstruct.reconstruct(tree, postproc) + +print(output) + +tree_new = python_parser3.parse(output) + +assert tree == tree_new diff --git a/lark-stubs/reconstruct.pyi b/lark-stubs/reconstruct.pyi index 2220c46..a30ef0d 100644 --- a/lark-stubs/reconstruct.pyi +++ b/lark-stubs/reconstruct.pyi @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- -from typing import List, Dict, Union +from typing import List, Dict, Union, Callable, Iterable + +from .grammar import Symbol from .lark import Lark from .tree import Tree from .visitors import Transformer_InPlace @@ -30,8 +32,8 @@ class MakeMatchTree: class Reconstructor: - def __init__(self, parser: Lark, term_subs: Dict[str, str] = ...): + def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]] = ...): ... - def reconstruct(self, tree: Tree) -> str: + def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]) -> str: ...