Merge pull request #846 from lark-parser/astutils

examples/advanced/create_ast.py

@@ -0,0 +1,105 @@
+"""
+    This example demonstrates how to transform a parse-tree into an AST using `lark.ast_utils`.
+
+    This example only works with Python 3.
+"""
+
+import sys
+from typing import List
+from dataclasses import dataclass
+
+from lark import Lark, ast_utils, Transformer, v_args
+
+this_module = sys.modules[__name__]
+
+
+#
+#   Define AST
+#
+class _Ast(ast_utils.Ast):
+    pass
+
+class _Statement(_Ast):
+    pass
+
+@dataclass
+class Value(_Ast):
+    value: object
+
+@dataclass
+class Name(_Ast):
+    name: str
+
+@dataclass
+class CodeBlock(_Ast, ast_utils.AsList):
+    statements: List[_Statement]
+
+@dataclass
+class If(_Statement):
+    cond: Value
+    then: CodeBlock
+
+@dataclass
+class SetVar(_Statement):
+    name: str
+    value: Value
+
+@dataclass
+class Print(_Statement):
+    value: Value
+
+
+class ToAst(Transformer):
+    def STRING(self, s):
+        # Remove quotation marks
+        return s[1:-1]
+
+    def DEC_NUMBER(self, n):
+        return int(n)
+
+    @v_args(inline=True)
+    def start(self, x):
+        return x
+
+#
+#   Define Parser
+#
+
+parser = Lark("""
+    start: code_block
+
+    code_block: statement+
+
+    ?statement: if | set_var | print
+
+    if: "if" value "{" code_block "}"
+    set_var: NAME "=" value ";"
+    print: "print" value ";"
+
+    value: name | STRING | DEC_NUMBER
+    name: NAME
+
+    %import python (NAME, STRING, DEC_NUMBER)
+    %import common.WS
+    %ignore WS
+    """,
+    parser="lalr",
+)
+
+transformer = ast_utils.create_transformer(this_module, ToAst())
+
+def parse(text):
+    return transformer.transform(parser.parse(text))
+
+#
+#   Test
+#
+
+if __name__ == '__main__':
+    print(parse("""
+        a = 1;
+        if a {
+            print "a is 1";
+            a = 2;
+        }
+    """))

lark-stubs/ast_utils.pyi

@@ -0,0 +1,17 @@
+import types
+from typing import Optional
+
+from .visitors import Transformer
+
+class Ast(object):
+    pass
+
+class AsList(object):
+    pass
+
+
+def create_transformer(
+        ast_module: types.ModuleType,
+        transformer: Optional[Transformer]=None
+) -> Transformer:
+    ...

lark/ast_utils.py

@@ -0,0 +1,51 @@
+"""
+    Module of utilities for transforming a lark.Tree into a custom Abstract Syntax Tree
+"""
+
+import inspect, re
+
+from lark import Transformer, v_args
+
+class Ast(object):
+    """Abstract class
+
+    Subclasses will be collected by `create_transformer()`
+    """
+    pass
+
+class AsList(object):
+    """Abstract class
+
+    Subclasses will be instanciated with the parse results as a single list, instead of as arguments.
+    """
+
+def camel_to_snake(name):
+    return re.sub(r'(?<!^)(?=[A-Z])', '_', name).lower()
+
+def _call(func, _data, children, _meta):
+    return func(*children)
+
+inline = v_args(wrapper=_call)
+
+def create_transformer(ast_module, transformer=None):
+    """Collects `Ast` subclasses from the given module, and creates a Lark transformer that builds the AST.
+
+    For each class, we create a corresponding rule in the transformer, with a matching name.
+    CamelCase names will be converted into snake_case. Example: "CodeBlock" -> "code_block".
+
+    Parameters:
+        ast_module - A Python module containing all the subclasses of `ast_utils.Ast`
+                     Classes starting with an underscore (`_`) will be skipped.
+        transformer (Optional[Transformer]) - An initial transformer. Its attributes may be overwritten.
+    """
+    t = transformer or Transformer()
+
+    for name, obj in inspect.getmembers(ast_module):
+        if not name.startswith('_') and inspect.isclass(obj):
+            if issubclass(obj, Ast):
+                if not issubclass(obj, AsList):
+                    obj = inline(obj).__get__(t)
+
+                setattr(t, camel_to_snake(name), obj)
+
+    return t