Added some docstrings, removed is_terminal from common

lark/common.py

@@ -7,9 +7,6 @@ Py36 = (sys.version_info[:2] >= (3, 6))
 
 
 ###{standalone
-def is_terminal(sym):
-    return sym.isupper()
-
 class GrammarError(Exception):
     pass
 

lark/lark.py

@@ -207,6 +207,7 @@ class Lark:
 
 
     def lex(self, text):
+        "Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'"
         if not hasattr(self, 'lexer'):
             self.lexer = self._build_lexer()
         stream = self.lexer.lex(text)
@@ -216,6 +217,7 @@ class Lark:
             return stream
 
     def parse(self, text):
+        "Parse the given text, according to the options provided. Returns a tree, unless specified otherwise."
         return self.parser.parse(text)
 
         # if self.profiler:

lark/lexer.py

@@ -3,7 +3,7 @@
 import re
 
 from .utils import Str, classify
-from .common import is_terminal, PatternStr, PatternRE, TokenDef
+from .common import PatternStr, PatternRE, TokenDef
 
 ###{standalone
 class LexError(Exception):

lark/load_grammar.py

@@ -11,7 +11,7 @@ from .lexer import Token, UnexpectedInput
 from .parse_tree_builder import ParseTreeBuilder
 from .parser_frontends import LALR
 from .parsers.lalr_parser import UnexpectedToken
-from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef
+from .common import GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef
 from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
 from .utils import classify, suppress
 
@@ -24,6 +24,9 @@ IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
 
 _RE_FLAGS = 'imslux'
 
+def is_terminal(sym):
+    return sym.isupper()
+
 _TERMINAL_NAMES = {
     '.' : 'DOT',
     ',' : 'COMMA',

lark/tree.py

@@ -45,6 +45,7 @@ class Tree(object):
 ###}
 
     def expand_kids_by_index(self, *indices):
+        "Expand (inline) children at the given indices"
         for i in sorted(indices, reverse=True): # reverse so that changing tail won't affect indices
             kid = self.children[i]
             self.children[i:i+1] = kid.children
@@ -62,9 +63,11 @@ class Tree(object):
         return hash((self.data, tuple(self.children)))
 
     def find_pred(self, pred):
+        "Find all nodes where pred(tree) == True"
         return filter(pred, self.iter_subtrees())
 
     def find_data(self, data):
+        "Find all nodes where tree.data == data"
         return self.find_pred(lambda t: t.data == data)
 
     def scan_values(self, pred):
@@ -108,10 +111,12 @@ class Tree(object):
         self.children = children
 
 class SlottedTree(Tree):
-    __slots__ = 'data', 'children', 'rule'
+    __slots__ = 'data', 'children', 'rule', '_meta'
 
 
 def pydot__tree_to_png(tree, filename):
+    "Creates a colorful image that represents the tree (data+children, without meta)"
+
     import pydot
     graph = pydot.Dot(graph_type='digraph', rankdir="LR")
 

lark/visitors.py

@@ -11,6 +11,14 @@ class Discard(Exception):
 # Transformers
 
 class Transformer:
+    """Visits the tree recursively, starting with the leaves and finally the root (bottom-up)
+
+    Calls its methods (provided by user via inheritance) according to tree.data
+    The returned value replaces the old one in the structure.
+
+    Can be used to implement map or reduce.
+    """
+
     def _call_userfunc(self, data, children, meta):
         # Assumes tree is already transformed
         try:
@@ -84,6 +92,7 @@ class TransformerChain(object):
 
 
 class Transformer_InPlace(Transformer):
+    "Non-recursive. Changes the tree in-place instead of returning new instances"
     def _transform_tree(self, tree):           # Cancel recursion
         return self._call_userfunc(tree.data, tree.children, tree.meta)
 
@@ -95,6 +104,7 @@ class Transformer_InPlace(Transformer):
 
 
 class Transformer_InPlaceRecursive(Transformer):
+    "Recursive. Changes the tree in-place instead of returning new instances"
     def _transform_tree(self, tree):
         tree.children = list(self._transform_children(tree.children))
         return self._call_userfunc(tree.data, tree.children, tree.meta)
@@ -113,7 +123,12 @@ class VisitorBase:
 
 
 class Visitor(VisitorBase):
-    "Bottom-up visitor"
+    """Bottom-up visitor, non-recursive
+
+    Visits the tree, starting with the leaves and finally the root (bottom-up)
+    Calls its methods (provided by user via inheritance) according to tree.data
+    """
+
 
     def visit(self, tree):
         for subtree in tree.iter_subtrees():
@@ -121,6 +136,12 @@ class Visitor(VisitorBase):
         return tree
 
 class Visitor_Recursive(VisitorBase):
+    """Bottom-up visitor, recursive
+
+    Visits the tree, starting with the leaves and finally the root (bottom-up)
+    Calls its methods (provided by user via inheritance) according to tree.data
+    """
+
     def visit(self, tree):
         for child in tree.children:
             if isinstance(child, Tree):
@@ -133,6 +154,7 @@ class Visitor_Recursive(VisitorBase):
 
 
 def visit_children_decor(func):
+    "See Interpreter"
     @wraps(func)
     def inner(cls, tree):
         values = cls.visit_children(tree)
@@ -141,8 +163,14 @@ def visit_children_decor(func):
 
 
 class Interpreter:
-    "Top-down visitor"
+    """Top-down visitor, recursive
+
+    Visits the tree, starting with the root and finally the leaves (top-down)
+    Calls its methods (provided by user via inheritance) according to tree.data
 
+    Unlike Transformer and Visitor, the Interpreter doesn't automatically visit its sub-branches.
+    The user has to explicitly call visit_children, or use the @visit_children_decor
+    """
     def visit(self, tree):
         return getattr(self, tree.data)(tree)
 
@@ -207,6 +235,7 @@ def _visitor_args_func_dec(func, inline=False, meta=False):
     return f
 
 def v_args(inline=False, meta=False):
+    "A convenience decorator factory, for modifying the behavior of user-supplied visitor methods"
     if inline and meta:
         raise ValueError("Visitor functions can either accept meta, or be inlined. Not both.")
     def _visitor_args_dec(obj):