diff --git a/examples/advanced/advanced_transformers.py b/examples/advanced/advanced_transformers.py new file mode 100644 index 0000000..9810f44 --- /dev/null +++ b/examples/advanced/advanced_transformers.py @@ -0,0 +1,67 @@ +""" +Transformer merging +================== + +This example is intended to show how transformers can be merged in order to +keep the individual steps clean and simple. + +.. note:: + The imported rules will have to be aliased according to the file it is in. + (See `storage.lark` for an implementation of this idea.) +""" +from lark import Lark, Tree +from json import dumps +from lark.visitors import Transformer, merge_transformers, v_args + +class JsonTreeToJson(Transformer): + @v_args(inline=True) + def string(self, s): + return s[1:-1].replace('\\"', '"') + + array = list + pair = tuple + object = dict + number = v_args(inline=True)(float) + + null = lambda self, _: None + true = lambda self, _: True + false = lambda self, _: False + +class CsvTreeToPandasDict(Transformer): + INT = int + FLOAT = float + SIGNED_FLOAT = float + WORD = str + NON_SEPARATOR_STRING = str + + def row(self, children): + return children + + def start(self, children): + data = {} + + header = children[0].children + for heading in header: + data[heading] = [] + + for row in children[1:]: + for i, element in enumerate(row): + data[header[i]].append(element) + + return data + +class Base(Transformer): + def start(self, children): + return children[0] + +if __name__ == "__main__": + merged = merge_transformers(Base(), csv=CsvTreeToPandasDict(), json=JsonTreeToJson()) + parser = Lark.open("storage.lark") + csv_tree = parser.parse("""# file lines author +data.json 12 Robin +data.csv 30 erezsh +compiler.py 123123 Megalng +""") + print("CSV data in pandas form:", merged.transform(csv_tree)) + json_tree = parser.parse(dumps({"test": "a", "dict": { "list": [1, 1.2] }})) + print("JSON data transformed: ", merged.transform(json_tree)) diff --git a/examples/advanced/csv.lark b/examples/advanced/csv.lark new file mode 100644 index 0000000..cc2b675 --- /dev/null +++ b/examples/advanced/csv.lark @@ -0,0 +1,14 @@ +start: header _NL row+ +header: "#" " "? (WORD _SEPARATOR?)+ +row: (_anything _SEPARATOR?)+ _NL +_anything: INT | WORD | NON_SEPARATOR_STRING | FLOAT | SIGNED_FLOAT +NON_SEPARATOR_STRING: /[a-zA-z.;\\\/]+/ +_SEPARATOR: /[ ]+/ + | "\t" + | "," + +%import common.NEWLINE -> _NL +%import common.WORD +%import common.INT +%import common.FLOAT +%import common.SIGNED_FLOAT diff --git a/examples/advanced/json.lark b/examples/advanced/json.lark new file mode 100644 index 0000000..bb77c35 --- /dev/null +++ b/examples/advanced/json.lark @@ -0,0 +1,19 @@ +?start: value + +?value: object + | array + | string + | SIGNED_NUMBER -> number + | "true" -> true + | "false" -> false + | "null" -> null + +array : "[" _WS? [value ("," _WS? value)*] "]" +object : "{" _WS? [pair ("," _WS? pair)*] "}" +pair : string ":" _WS value + +string : ESCAPED_STRING + +%import common.ESCAPED_STRING +%import common.SIGNED_NUMBER +%import common.WS -> _WS diff --git a/examples/advanced/storage.lark b/examples/advanced/storage.lark new file mode 100644 index 0000000..64718ed --- /dev/null +++ b/examples/advanced/storage.lark @@ -0,0 +1,8 @@ +start: csv__start + | json__start + +// Renaming of the import variables is required, as they +// receive the namespace of this file. +// See: https://github.com/lark-parser/lark/pull/973#issuecomment-907287565 +%import .csv.start -> csv__start +%import .json.start -> json__start diff --git a/lark/visitors.py b/lark/visitors.py index 4894cab..498a676 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -151,9 +151,16 @@ class Transformer(_Decoratable): def merge_transformers(base_transformer=None, **kwargs): """ - Add the methods of other transformer to this one. + Paramaters: + :param base_transformer: Transformer that all other transformers will be added to. + :param \**kwargs: Key-value arguments providing the prefix for the methods of the transformer and the Transformers themselves. - This method is meant to aid in the maintenance of imports. + Compose a new transformer from a base and the in the `**kwargs` provided Transformer instances. + + The key should match the grammar file that the Transformer is supposed to manipulate. + + This method is meant to aid the composing of large transformers that + manipulate grammars that cross multiple lark files. Example: ```python @@ -183,11 +190,12 @@ def merge_transformers(base_transformer=None, **kwargs): In the above code block `regular_transformer` and `composed_transformer` should behave identically. """ + infix = "__" if base_transformer is None: base_transformer = Transformer() for prefix, transformer in kwargs.items(): - prefix += "__" + prefix += infix for method_name in dir(transformer): method = getattr(transformer, method_name)