Added advanced example for merge_transformers

examples/advanced/advanced_transformers.py

@@ -0,0 +1,67 @@
+"""
+Transformer merging
+==================
+
+This example is intended to show how transformers can be merged in order to
+keep the individual steps clean and simple.
+
+.. note::
+    The imported rules will have to be aliased according to the file it is in.
+    (See `storage.lark` for an implementation of this idea.)
+"""
+from lark import Lark, Tree
+from json import dumps
+from lark.visitors import Transformer, merge_transformers, v_args
+
+class JsonTreeToJson(Transformer):
+    @v_args(inline=True)
+    def string(self, s):
+        return s[1:-1].replace('\\"', '"')
+
+    array = list
+    pair = tuple
+    object = dict
+    number = v_args(inline=True)(float)
+
+    null = lambda self, _: None
+    true = lambda self, _: True
+    false = lambda self, _: False
+
+class CsvTreeToPandasDict(Transformer):
+    INT = int
+    FLOAT = float
+    SIGNED_FLOAT = float
+    WORD = str
+    NON_SEPARATOR_STRING = str
+
+    def row(self, children):
+        return children
+
+    def start(self, children):
+        data = {}
+
+        header = children[0].children
+        for heading in header:
+            data[heading] = []
+
+        for row in children[1:]:
+            for i, element in enumerate(row):
+                data[header[i]].append(element)
+
+        return data
+
+class Base(Transformer):
+    def start(self, children):
+        return children[0]
+
+if __name__ == "__main__":
+    merged = merge_transformers(Base(), csv=CsvTreeToPandasDict(), json=JsonTreeToJson())
+    parser = Lark.open("storage.lark")
+    csv_tree = parser.parse("""# file lines author
+data.json 12 Robin
+data.csv  30 erezsh
+compiler.py 123123 Megalng
+""")
+    print("CSV data in pandas form:", merged.transform(csv_tree))
+    json_tree = parser.parse(dumps({"test": "a", "dict": { "list": [1, 1.2] }}))
+    print("JSON data transformed: ", merged.transform(json_tree))

examples/advanced/csv.lark

@@ -0,0 +1,14 @@
+start: header _NL row+
+header: "#" " "? (WORD _SEPARATOR?)+
+row: (_anything _SEPARATOR?)+ _NL
+_anything: INT | WORD | NON_SEPARATOR_STRING | FLOAT | SIGNED_FLOAT
+NON_SEPARATOR_STRING: /[a-zA-z.;\\\/]+/
+_SEPARATOR: /[  ]+/
+          | "\t"
+          | ","
+
+%import common.NEWLINE -> _NL
+%import common.WORD
+%import common.INT
+%import common.FLOAT
+%import common.SIGNED_FLOAT

examples/advanced/json.lark

@@ -0,0 +1,19 @@
+?start: value
+
+?value: object
+      | array
+      | string
+      | SIGNED_NUMBER      -> number
+      | "true"             -> true
+      | "false"            -> false
+      | "null"             -> null
+
+array  : "[" _WS? [value ("," _WS? value)*] "]"
+object : "{" _WS? [pair ("," _WS? pair)*] "}"
+pair   : string ":" _WS value
+
+string : ESCAPED_STRING
+
+%import common.ESCAPED_STRING
+%import common.SIGNED_NUMBER
+%import common.WS -> _WS

examples/advanced/storage.lark

@@ -0,0 +1,8 @@
+start: csv__start
+     | json__start
+
+// Renaming of the import variables is required, as they
+// receive the namespace of this file.
+// See: https://github.com/lark-parser/lark/pull/973#issuecomment-907287565
+%import .csv.start -> csv__start
+%import .json.start -> json__start

lark/visitors.py

@@ -151,9 +151,16 @@ class Transformer(_Decoratable):
 
 def merge_transformers(base_transformer=None, **kwargs):
     """
-    Add the methods of other transformer to this one.
+    Paramaters:
+        :param base_transformer: Transformer that all other transformers will be added to.
+        :param \**kwargs: Key-value arguments providing the prefix for the methods of the transformer and the Transformers themselves.
 
-    This method is meant to aid in the maintenance of imports.
+    Compose a new transformer from a base and the in the `**kwargs` provided Transformer instances.
+
+    The key should match the grammar file that the Transformer is supposed to manipulate.
+
+    This method is meant to aid the composing of large transformers that
+    manipulate grammars that cross multiple lark files.
 
     Example:
     ```python
@@ -183,11 +190,12 @@ def merge_transformers(base_transformer=None, **kwargs):
     In the above code block `regular_transformer` and `composed_transformer`
     should behave identically.
     """
+    infix = "__"
 
     if base_transformer is None:
         base_transformer = Transformer()
     for prefix, transformer in kwargs.items():
-        prefix += "__"
+        prefix += infix
 
         for method_name in dir(transformer):
             method = getattr(transformer, method_name)