@@ -0,0 +1,67 @@ | |||||
""" | |||||
Transformer merging | |||||
================== | |||||
This example is intended to show how transformers can be merged in order to | |||||
keep the individual steps clean and simple. | |||||
.. note:: | |||||
The imported rules will have to be aliased according to the file it is in. | |||||
(See `storage.lark` for an implementation of this idea.) | |||||
""" | |||||
from lark import Lark, Tree | |||||
from json import dumps | |||||
from lark.visitors import Transformer, merge_transformers, v_args | |||||
class JsonTreeToJson(Transformer): | |||||
@v_args(inline=True) | |||||
def string(self, s): | |||||
return s[1:-1].replace('\\"', '"') | |||||
array = list | |||||
pair = tuple | |||||
object = dict | |||||
number = v_args(inline=True)(float) | |||||
null = lambda self, _: None | |||||
true = lambda self, _: True | |||||
false = lambda self, _: False | |||||
class CsvTreeToPandasDict(Transformer): | |||||
INT = int | |||||
FLOAT = float | |||||
SIGNED_FLOAT = float | |||||
WORD = str | |||||
NON_SEPARATOR_STRING = str | |||||
def row(self, children): | |||||
return children | |||||
def start(self, children): | |||||
data = {} | |||||
header = children[0].children | |||||
for heading in header: | |||||
data[heading] = [] | |||||
for row in children[1:]: | |||||
for i, element in enumerate(row): | |||||
data[header[i]].append(element) | |||||
return data | |||||
class Base(Transformer): | |||||
def start(self, children): | |||||
return children[0] | |||||
if __name__ == "__main__": | |||||
merged = merge_transformers(Base(), csv=CsvTreeToPandasDict(), json=JsonTreeToJson()) | |||||
parser = Lark.open("storage.lark") | |||||
csv_tree = parser.parse("""# file lines author | |||||
data.json 12 Robin | |||||
data.csv 30 erezsh | |||||
compiler.py 123123 Megalng | |||||
""") | |||||
print("CSV data in pandas form:", merged.transform(csv_tree)) | |||||
json_tree = parser.parse(dumps({"test": "a", "dict": { "list": [1, 1.2] }})) | |||||
print("JSON data transformed: ", merged.transform(json_tree)) |
@@ -0,0 +1,14 @@ | |||||
start: header _NL row+ | |||||
header: "#" " "? (WORD _SEPARATOR?)+ | |||||
row: (_anything _SEPARATOR?)+ _NL | |||||
_anything: INT | WORD | NON_SEPARATOR_STRING | FLOAT | SIGNED_FLOAT | |||||
NON_SEPARATOR_STRING: /[a-zA-z.;\\\/]+/ | |||||
_SEPARATOR: /[ ]+/ | |||||
| "\t" | |||||
| "," | |||||
%import common.NEWLINE -> _NL | |||||
%import common.WORD | |||||
%import common.INT | |||||
%import common.FLOAT | |||||
%import common.SIGNED_FLOAT |
@@ -0,0 +1,19 @@ | |||||
?start: value | |||||
?value: object | |||||
| array | |||||
| string | |||||
| SIGNED_NUMBER -> number | |||||
| "true" -> true | |||||
| "false" -> false | |||||
| "null" -> null | |||||
array : "[" _WS? [value ("," _WS? value)*] "]" | |||||
object : "{" _WS? [pair ("," _WS? pair)*] "}" | |||||
pair : string ":" _WS value | |||||
string : ESCAPED_STRING | |||||
%import common.ESCAPED_STRING | |||||
%import common.SIGNED_NUMBER | |||||
%import common.WS -> _WS |
@@ -0,0 +1,8 @@ | |||||
start: csv__start | |||||
| json__start | |||||
// Renaming of the import variables is required, as they | |||||
// receive the namespace of this file. | |||||
// See: https://github.com/lark-parser/lark/pull/973#issuecomment-907287565 | |||||
%import .csv.start -> csv__start | |||||
%import .json.start -> json__start |
@@ -151,9 +151,16 @@ class Transformer(_Decoratable): | |||||
def merge_transformers(base_transformer=None, **kwargs): | def merge_transformers(base_transformer=None, **kwargs): | ||||
""" | """ | ||||
Add the methods of other transformer to this one. | |||||
Paramaters: | |||||
:param base_transformer: Transformer that all other transformers will be added to. | |||||
:param \**kwargs: Key-value arguments providing the prefix for the methods of the transformer and the Transformers themselves. | |||||
This method is meant to aid in the maintenance of imports. | |||||
Compose a new transformer from a base and the in the `**kwargs` provided Transformer instances. | |||||
The key should match the grammar file that the Transformer is supposed to manipulate. | |||||
This method is meant to aid the composing of large transformers that | |||||
manipulate grammars that cross multiple lark files. | |||||
Example: | Example: | ||||
```python | ```python | ||||
@@ -183,11 +190,12 @@ def merge_transformers(base_transformer=None, **kwargs): | |||||
In the above code block `regular_transformer` and `composed_transformer` | In the above code block `regular_transformer` and `composed_transformer` | ||||
should behave identically. | should behave identically. | ||||
""" | """ | ||||
infix = "__" | |||||
if base_transformer is None: | if base_transformer is None: | ||||
base_transformer = Transformer() | base_transformer = Transformer() | ||||
for prefix, transformer in kwargs.items(): | for prefix, transformer in kwargs.items(): | ||||
prefix += "__" | |||||
prefix += infix | |||||
for method_name in dir(transformer): | for method_name in dir(transformer): | ||||
method = getattr(transformer, method_name) | method = getattr(transformer, method_name) | ||||