Added example + Corrected python3.lark

examples/advanced/python3.lark

@@ -23,9 +23,11 @@ decorated: decorators (classdef | funcdef | async_funcdef)
 async_funcdef: "async" funcdef
 funcdef: "def" NAME "(" parameters? ")" ["->" test] ":" suite
 
-parameters: paramvalue ("," paramvalue)* ["," "/"] ["," [starparams | kwparams]]
+parameters: paramvalue ("," paramvalue)* ["," SLASH] ["," [starparams | kwparams]]
           | starparams
           | kwparams
+
+SLASH: "/" // Otherwise the it will completely disappear and it will be undisguisable in the result
 starparams: "*" typedparam? ("," paramvalue)* ["," kwparams]
 kwparams: "**" typedparam
 
@@ -116,9 +118,10 @@ AWAIT: "await"
           | atom_expr "." NAME               -> getattr
           | atom
 
-?atom: "(" [yield_expr|testlist_comp] ")" -> tuple
+?atom: "(" [yield_expr|tuplelist_comp] ")" -> tuple
      | "[" [testlist_comp] "]"  -> list
-     | "{" [dictorsetmaker] "}" -> dict
+     | "{" [dict_comp] "}" -> dict
+     | "{" set_comp "}" -> set
      | NAME -> var
      | number | string+
      | "(" test ")"
@@ -127,13 +130,23 @@ AWAIT: "await"
      | "True"    -> const_true
      | "False"   -> const_false
 
-?testlist_comp: (test|star_expr) [comp_for | ("," (test|star_expr))+ [","] | ","]
-subscriptlist: subscript ("," subscript)* [","]
+?testlist_comp: test | tuplelist_comp
+tuplelist_comp: (test|star_expr) (comp_for | ("," (test|star_expr))+ [","] | ",")
+?subscriptlist: subscript
+              | subscript (("," subscript)+ [","] | ",") -> subscript_tuple
 subscript: test | [test] ":" [test] [sliceop]
 sliceop: ":" [test]
-exprlist: (expr|star_expr) ("," (expr|star_expr))* [","]
-testlist: test ("," test)* [","]
-dictorsetmaker: ( ((test ":" test | "**" expr) (comp_for | ("," (test ":" test | "**" expr))* [","])) | ((test | star_expr) (comp_for | ("," (test | star_expr))* [","])) )
+exprlist: (expr|star_expr)
+        | (expr|star_expr) (("," (expr|star_expr))+ [","]|",") -> exprlist_tuple
+testlist: test | testlist_tuple
+testlist_tuple: test (("," test)+ [","] | ",")
+dict_comp: key_value comp_for 
+         | (key_value | "**" expr) ("," (key_value | "**" expr))* [","]
+
+key_value: test ":"  test
+
+set_comp: test comp_for 
+        | (test|star_expr) ("," (test | star_expr))* [","]
 
 classdef: "class" NAME ["(" [arguments] ")"] ":" suite
 

examples/advanced/reconstruct_python.py

@@ -0,0 +1,58 @@
+from lark import Token
+from lark.reconstruct import Reconstructor
+
+from python_parser import python_parser3
+
+
+test_python = open(__file__).read()
+
+def special(sym):
+    return Token('SPECIAL', sym.name)
+
+SPACE_AFTER = set(',+-*/~@<>="|:')
+SPACE_BEFORE = (SPACE_AFTER - set(',:')) | set('\'')
+
+def postproc(items):
+    stack = ['\n']
+    actions = []
+    last_was_whitespace = True
+    for item in items:
+        if isinstance(item, Token) and item.type == 'SPECIAL': 
+            actions.append(item.value)
+        else:
+            if actions:
+                assert actions[0] == '_NEWLINE' and '_NEWLINE' not in actions[1:], actions
+
+                for a in actions[1:]:
+                    if a == '_INDENT':
+                        stack.append(stack[-1] + ' ' * 4)
+                    else:
+                        assert a == '_DEDENT'
+                        stack.pop()
+                actions.clear()
+                yield stack[-1]
+                last_was_whitespace = True
+            if not last_was_whitespace:
+                if item[0] in SPACE_BEFORE:
+                    yield ' '
+            yield item
+            last_was_whitespace = item[-1].isspace()
+            if not last_was_whitespace:
+                if item[-1] in SPACE_AFTER:
+                    yield ' '
+                    last_was_whitespace = True
+    yield "\n"
+
+
+tree = python_parser3.parse(test_python)
+
+
+python_reconstruct = Reconstructor(python_parser3, {'_NEWLINE': special, '_DEDENT': special, '_INDENT': special})
+
+output = python_reconstruct.reconstruct(tree, postproc)
+
+print(output)
+
+tree_new = python_parser3.parse(output)
+
+assert tree == tree_new

lark-stubs/reconstruct.pyi

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Callable, Iterable
+
+from .grammar import Symbol
 from .lark import Lark
 from .tree import Tree
 from .visitors import Transformer_InPlace
@@ -30,8 +32,8 @@ class MakeMatchTree:
 
 class Reconstructor:
 
-    def __init__(self, parser: Lark, term_subs: Dict[str, str] = ...):
+    def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]] = ...):
         ...
 
-    def reconstruct(self, tree: Tree) -> str:
+    def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]) -> str:
         ...