diff --git a/lark/common.py b/lark/common.py index 84a4139..78ef205 100644 --- a/lark/common.py +++ b/lark/common.py @@ -7,9 +7,6 @@ Py36 = (sys.version_info[:2] >= (3, 6)) ###{standalone -def is_terminal(sym): - return sym.isupper() - class GrammarError(Exception): pass diff --git a/lark/lark.py b/lark/lark.py index 4fc0062..bb15a2f 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -207,6 +207,7 @@ class Lark: def lex(self, text): + "Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'" if not hasattr(self, 'lexer'): self.lexer = self._build_lexer() stream = self.lexer.lex(text) @@ -216,6 +217,7 @@ class Lark: return stream def parse(self, text): + "Parse the given text, according to the options provided. Returns a tree, unless specified otherwise." return self.parser.parse(text) # if self.profiler: diff --git a/lark/lexer.py b/lark/lexer.py index cf7ad30..e332e22 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -3,7 +3,7 @@ import re from .utils import Str, classify -from .common import is_terminal, PatternStr, PatternRE, TokenDef +from .common import PatternStr, PatternRE, TokenDef ###{standalone class LexError(Exception): diff --git a/lark/load_grammar.py b/lark/load_grammar.py index bbfedf9..bd6fa36 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -11,7 +11,7 @@ from .lexer import Token, UnexpectedInput from .parse_tree_builder import ParseTreeBuilder from .parser_frontends import LALR from .parsers.lalr_parser import UnexpectedToken -from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef +from .common import GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol from .utils import classify, suppress @@ -24,6 +24,9 @@ IMPORT_PATHS = [os.path.join(__path__, 'grammars')] _RE_FLAGS = 'imslux' +def is_terminal(sym): + return sym.isupper() + _TERMINAL_NAMES = { '.' : 'DOT', ',' : 'COMMA', diff --git a/lark/tree.py b/lark/tree.py index 1490632..000823e 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -45,6 +45,7 @@ class Tree(object): ###} def expand_kids_by_index(self, *indices): + "Expand (inline) children at the given indices" for i in sorted(indices, reverse=True): # reverse so that changing tail won't affect indices kid = self.children[i] self.children[i:i+1] = kid.children @@ -62,9 +63,11 @@ class Tree(object): return hash((self.data, tuple(self.children))) def find_pred(self, pred): + "Find all nodes where pred(tree) == True" return filter(pred, self.iter_subtrees()) def find_data(self, data): + "Find all nodes where tree.data == data" return self.find_pred(lambda t: t.data == data) def scan_values(self, pred): @@ -108,10 +111,12 @@ class Tree(object): self.children = children class SlottedTree(Tree): - __slots__ = 'data', 'children', 'rule' + __slots__ = 'data', 'children', 'rule', '_meta' def pydot__tree_to_png(tree, filename): + "Creates a colorful image that represents the tree (data+children, without meta)" + import pydot graph = pydot.Dot(graph_type='digraph', rankdir="LR") diff --git a/lark/visitors.py b/lark/visitors.py index 2f87c25..752df69 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -11,6 +11,14 @@ class Discard(Exception): # Transformers class Transformer: + """Visits the tree recursively, starting with the leaves and finally the root (bottom-up) + + Calls its methods (provided by user via inheritance) according to tree.data + The returned value replaces the old one in the structure. + + Can be used to implement map or reduce. + """ + def _call_userfunc(self, data, children, meta): # Assumes tree is already transformed try: @@ -84,6 +92,7 @@ class TransformerChain(object): class Transformer_InPlace(Transformer): + "Non-recursive. Changes the tree in-place instead of returning new instances" def _transform_tree(self, tree): # Cancel recursion return self._call_userfunc(tree.data, tree.children, tree.meta) @@ -95,6 +104,7 @@ class Transformer_InPlace(Transformer): class Transformer_InPlaceRecursive(Transformer): + "Recursive. Changes the tree in-place instead of returning new instances" def _transform_tree(self, tree): tree.children = list(self._transform_children(tree.children)) return self._call_userfunc(tree.data, tree.children, tree.meta) @@ -113,7 +123,12 @@ class VisitorBase: class Visitor(VisitorBase): - "Bottom-up visitor" + """Bottom-up visitor, non-recursive + + Visits the tree, starting with the leaves and finally the root (bottom-up) + Calls its methods (provided by user via inheritance) according to tree.data + """ + def visit(self, tree): for subtree in tree.iter_subtrees(): @@ -121,6 +136,12 @@ class Visitor(VisitorBase): return tree class Visitor_Recursive(VisitorBase): + """Bottom-up visitor, recursive + + Visits the tree, starting with the leaves and finally the root (bottom-up) + Calls its methods (provided by user via inheritance) according to tree.data + """ + def visit(self, tree): for child in tree.children: if isinstance(child, Tree): @@ -133,6 +154,7 @@ class Visitor_Recursive(VisitorBase): def visit_children_decor(func): + "See Interpreter" @wraps(func) def inner(cls, tree): values = cls.visit_children(tree) @@ -141,8 +163,14 @@ def visit_children_decor(func): class Interpreter: - "Top-down visitor" + """Top-down visitor, recursive + + Visits the tree, starting with the root and finally the leaves (top-down) + Calls its methods (provided by user via inheritance) according to tree.data + Unlike Transformer and Visitor, the Interpreter doesn't automatically visit its sub-branches. + The user has to explicitly call visit_children, or use the @visit_children_decor + """ def visit(self, tree): return getattr(self, tree.data)(tree) @@ -207,6 +235,7 @@ def _visitor_args_func_dec(func, inline=False, meta=False): return f def v_args(inline=False, meta=False): + "A convenience decorator factory, for modifying the behavior of user-supplied visitor methods" if inline and meta: raise ValueError("Visitor functions can either accept meta, or be inlined. Not both.") def _visitor_args_dec(obj):