diff --git a/lark-stubs/load_grammar.pyi b/lark-stubs/load_grammar.pyi deleted file mode 100644 index 86a6341..0000000 --- a/lark-stubs/load_grammar.pyi +++ /dev/null @@ -1,31 +0,0 @@ -from typing import List, Tuple, Union, Callable, Dict, Optional - -from .tree import Tree -from .grammar import RuleOptions -from .exceptions import UnexpectedInput - - -class Grammar: - rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]] - term_defs: List[Tuple[str, Tuple[Tree, int]]] - ignore: List[str] - - -class GrammarBuilder: - global_keep_all_tokens: bool - import_paths: List[Union[str, Callable]] - used_files: Dict[str, str] - - def __init__(self, global_keep_all_tokens: bool = False, import_paths: List[Union[str, Callable]] = None, used_files: Dict[str, str]=None) -> None: ... - - def load_grammar(self, grammar_text: str, grammar_name: str = ..., mangle: Callable[[str], str] = None) -> None: ... - - def do_import(self, dotted_path: Tuple[str, ...], base_path: Optional[str], aliases: Dict[str, str], - base_mangle: Callable[[str], str] = None) -> None: ... - - def validate(self) -> None: ... - - def build(self) -> Grammar: ... - - -def find_grammar_errors(text: str, start: str='start') -> List[Tuple[UnexpectedInput, str]]: ... \ No newline at end of file diff --git a/lark-stubs/reconstruct.pyi b/lark-stubs/reconstruct.pyi deleted file mode 100644 index a8d39e3..0000000 --- a/lark-stubs/reconstruct.pyi +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- - -from typing import List, Dict, Union, Callable, Iterable - -from .grammar import Symbol -from .lark import Lark -from .tree import Tree -from .visitors import Transformer_InPlace -from .lexer import TerminalDef - - -class WriteTokensTransformer(Transformer_InPlace): - - def __init__(self, tokens: Dict[str, TerminalDef], term_subs: Dict[str, Callable[[Symbol], str]] = ...): ... - - -class MatchTree(Tree): - pass - - -class MakeMatchTree: - name: str - expansion: List[TerminalDef] - - def __init__(self, name: str, expansion: List[TerminalDef]): - ... - - def __call__(self, args: List[Union[str, Tree]]): - ... - - -class Reconstructor: - - def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]] = ...): - ... - - def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]=None, - insert_spaces: bool = True) -> str: - ... diff --git a/lark-stubs/visitors.pyi b/lark-stubs/visitors.pyi deleted file mode 100644 index 3a934ee..0000000 --- a/lark-stubs/visitors.pyi +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- - -from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union -from abc import ABC -from .tree import Tree - -_T = TypeVar('_T') -_R = TypeVar('_R') -_FUNC = Callable[..., _T] -_DECORATED = Union[_FUNC, type] - - -class Transformer(ABC, Generic[_T]): - - def __init__(self, visit_tokens: bool = True) -> None: - ... - - def transform(self, tree: Tree) -> _T: - ... - - def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]: - ... - - -class TransformerChain(Generic[_T]): - transformers: Tuple[Transformer[_T], ...] - - def __init__(self, *transformers: Transformer[_T]) -> None: - ... - - def transform(self, tree: Tree) -> _T: - ... - - def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]: - ... - - -class Transformer_InPlace(Transformer): - pass - - -class Transformer_NonRecursive(Transformer): - pass - - -class Transformer_InPlaceRecursive(Transformer): - pass - - -class VisitorBase: - pass - - -class Visitor(VisitorBase, ABC, Generic[_T]): - - def visit(self, tree: Tree) -> Tree: - ... - - def visit_topdown(self, tree: Tree) -> Tree: - ... - - -class Visitor_Recursive(VisitorBase): - - def visit(self, tree: Tree) -> Tree: - ... - - def visit_topdown(self, tree: Tree) -> Tree: - ... - - -class Interpreter(ABC, Generic[_T]): - - def visit(self, tree: Tree) -> _T: - ... - - def visit_children(self, tree: Tree) -> List[_T]: - ... - - -_InterMethod = Callable[[Type[Interpreter], _T], _R] - - -def v_args( - inline: bool = False, - meta: bool = False, - tree: bool = False, - wrapper: Callable = None -) -> Callable[[_DECORATED], _DECORATED]: - ... - - -def visit_children_decor(func: _InterMethod) -> _InterMethod: - ... - - -class Discard(Exception): - pass - - -# Deprecated -class InlineTransformer: - pass - - -# Deprecated -def inline_args(obj: _FUNC) -> _FUNC: - ... diff --git a/lark/indenter.py b/lark/indenter.py index 496f6e7..b6f47d6 100644 --- a/lark/indenter.py +++ b/lark/indenter.py @@ -1,13 +1,14 @@ "Provides Indentation services for languages with indentation similar to Python" from abc import ABC, abstractmethod -from typing import Tuple, List, Iterator, Optional from .exceptions import LarkError from .lark import PostLex from .lexer import Token ###{standalone +from typing import Tuple, List, Iterator, Optional + class DedentError(LarkError): pass diff --git a/lark/load_grammar.py b/lark/load_grammar.py index cb8856b..9ee3691 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -8,6 +8,7 @@ import pkgutil from ast import literal_eval from numbers import Integral from contextlib import suppress +from typing import List, Tuple, Union, Callable, Dict, Optional from .utils import bfs, Py36, logger, classify_bool, is_id_continue, is_id_start, bfs_all_unique from .lexer import Token, TerminalDef, PatternStr, PatternRE @@ -17,7 +18,7 @@ from .parser_frontends import ParsingFrontend from .common import LexerConf, ParserConf from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol from .utils import classify, dedup_list -from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError +from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError, UnexpectedInput from .tree import Tree, SlottedTree as ST from .visitors import Transformer, Visitor, v_args, Transformer_InPlace, Transformer_NonRecursive @@ -552,9 +553,9 @@ def nr_deepcopy_tree(t): class Grammar: def __init__(self, rule_defs, term_defs, ignore): - self.term_defs = term_defs - self.rule_defs = rule_defs - self.ignore = ignore + self.term_defs: List[Tuple[str, Tuple[Tree, int]]] = term_defs + self.rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]] = rule_defs + self.ignore: List[str] = ignore def compile(self, start, terminals_to_keep): # We change the trees in-place (to support huge grammars) @@ -874,7 +875,7 @@ def _search_interactive_parser(interactive_parser, predicate): if predicate(p): return path, p -def find_grammar_errors(text, start='start'): +def find_grammar_errors(text: str, start: str='start') -> List[Tuple[UnexpectedInput, str]]: errors = [] def on_error(e): errors.append((e, _error_repr(e))) @@ -923,10 +924,10 @@ def _mangle_exp(exp, mangle): class GrammarBuilder: - def __init__(self, global_keep_all_tokens=False, import_paths=None, used_files=None): - self.global_keep_all_tokens = global_keep_all_tokens - self.import_paths = import_paths or [] - self.used_files = used_files or {} + def __init__(self, global_keep_all_tokens: bool=False, import_paths: List[Union[str, Callable]]=None, used_files: Dict[str, str]=None) -> None: + self.global_keep_all_tokens: bool = global_keep_all_tokens + self.import_paths: List[Union[str, Callable]] = import_paths or [] + self.used_files: Dict[str, str] = used_files or {} self._definitions = {} self._ignore_names = [] @@ -1067,7 +1068,7 @@ class GrammarBuilder: return name, exp, params, opts - def load_grammar(self, grammar_text, grammar_name="", mangle=None): + def load_grammar(self, grammar_text: str, grammar_name: str="", mangle: Callable[[str], str]=None) -> None: tree = _parse_grammar(grammar_text, grammar_name) imports = {} @@ -1130,7 +1131,7 @@ class GrammarBuilder: self._definitions = {k: v for k, v in self._definitions.items() if k in _used} - def do_import(self, dotted_path, base_path, aliases, base_mangle=None): + def do_import(self, dotted_path: Tuple[str, ...], base_path: Optional[str], aliases: Dict[str, str], base_mangle: Callable[[str], str]=None) -> None: assert dotted_path mangle = _get_mangle('__'.join(dotted_path), aliases, base_mangle) grammar_path = os.path.join(*dotted_path) + EXT @@ -1166,7 +1167,7 @@ class GrammarBuilder: assert False, "Couldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for it" % (dotted_path,) - def validate(self): + def validate(self) -> None: for name, (params, exp, _options) in self._definitions.items(): for i, p in enumerate(params): if p in self._definitions: @@ -1195,7 +1196,7 @@ class GrammarBuilder: if not set(self._definitions).issuperset(self._ignore_names): raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(self._ignore_names) - set(self._definitions))) - def build(self): + def build(self) -> Grammar: self.validate() rule_defs = [] term_defs = [] diff --git a/lark/reconstruct.py b/lark/reconstruct.py index ab2fb38..2f0911b 100644 --- a/lark/reconstruct.py +++ b/lark/reconstruct.py @@ -1,11 +1,13 @@ """Reconstruct text from a tree, based on Lark grammar""" +from typing import List, Dict, Union, Callable, Iterable import unicodedata +from .lark import Lark from .tree import Tree from .visitors import Transformer_InPlace -from .lexer import Token, PatternStr -from .grammar import Terminal, NonTerminal +from .lexer import Token, PatternStr, TerminalDef +from .grammar import Terminal, NonTerminal, Symbol from .tree_matcher import TreeMatcher, is_discarded_terminal from .utils import is_id_continue @@ -21,7 +23,7 @@ def is_iter_empty(i): class WriteTokensTransformer(Transformer_InPlace): "Inserts discarded tokens into their correct place, according to the rules of grammar" - def __init__(self, tokens, term_subs): + def __init__(self, tokens: Dict[str, TerminalDef], term_subs: Dict[str, Callable[[Symbol], str]]) -> None: self.tokens = tokens self.term_subs = term_subs @@ -70,7 +72,7 @@ class Reconstructor(TreeMatcher): term_subs: a dictionary of [Terminal name as str] to [output text as str] """ - def __init__(self, parser, term_subs=None): + def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]]=None) -> None: TreeMatcher.__init__(self, parser) self.write_tokens = WriteTokensTransformer({t.name:t for t in self.tokens}, term_subs or {}) @@ -87,7 +89,7 @@ class Reconstructor(TreeMatcher): else: yield item - def reconstruct(self, tree, postproc=None, insert_spaces=True): + def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]=None, insert_spaces: bool=True) -> str: x = self._reconstruct(tree) if postproc: x = postproc(x) diff --git a/lark/visitors.py b/lark/visitors.py index 22beb47..847c468 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -1,3 +1,4 @@ +from abc import ABC from functools import wraps from .utils import smart_decorator, combine_alternatives @@ -7,7 +8,12 @@ from .lexer import Token ###{standalone from inspect import getmembers, getmro +from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union +_T = TypeVar('_T') +_R = TypeVar('_R') +_FUNC = Callable[..., _T] +_DECORATED = Union[_FUNC, type] class Discard(Exception): """When raising the Discard exception in a transformer callback, @@ -46,7 +52,7 @@ class _Decoratable: return cls -class Transformer(_Decoratable): +class Transformer(_Decoratable, ABC, Generic[_T]): """Transformers visit each node of the tree, and run the appropriate method on it according to the node's data. Methods are provided by the user via inheritance, and called according to ``tree.data``. @@ -74,7 +80,7 @@ class Transformer(_Decoratable): """ __visit_tokens__ = True # For backwards compatibility - def __init__(self, visit_tokens=True): + def __init__(self, visit_tokens: bool=True) -> None: self.__visit_tokens__ = visit_tokens def _call_userfunc(self, tree, new_children=None): @@ -125,11 +131,11 @@ class Transformer(_Decoratable): children = list(self._transform_children(tree.children)) return self._call_userfunc(tree, children) - def transform(self, tree): + def transform(self, tree: Tree) -> _T: "Transform the given tree, and return the final result" return self._transform_tree(tree) - def __mul__(self, other): + def __mul__(self, other: 'Transformer[_T]') -> 'TransformerChain[_T]': """Chain two transformers together, returning a new transformer. """ return TransformerChain(self, other) @@ -149,16 +155,16 @@ class Transformer(_Decoratable): return token -class TransformerChain(object): +class TransformerChain(Generic[_T]): def __init__(self, *transformers): - self.transformers = transformers + self.transformers: Tuple[Transformer[_T], ...] = transformers - def transform(self, tree): + def transform(self, tree: Tree) -> _T: for t in self.transformers: tree = t.transform(tree) return tree - def __mul__(self, other): + def __mul__(self, other: Transformer[_T]) -> 'TransformerChain[_T]': return TransformerChain(*self.transformers + (other,)) @@ -239,19 +245,19 @@ class VisitorBase: return cls -class Visitor(VisitorBase): +class Visitor(VisitorBase, ABC, Generic[_T]): """Tree visitor, non-recursive (can handle huge trees). Visiting a node calls its methods (provided by the user via inheritance) according to ``tree.data`` """ - def visit(self, tree): + def visit(self, tree: Tree) -> Tree: "Visits the tree, starting with the leaves and finally the root (bottom-up)" for subtree in tree.iter_subtrees(): self._call_userfunc(subtree) return tree - def visit_topdown(self,tree): + def visit_topdown(self, tree: Tree) -> Tree: "Visit the tree, starting at the root, and ending at the leaves (top-down)" for subtree in tree.iter_subtrees_topdown(): self._call_userfunc(subtree) @@ -266,7 +272,7 @@ class Visitor_Recursive(VisitorBase): Slightly faster than the non-recursive version. """ - def visit(self, tree): + def visit(self, tree: Tree) -> Tree: "Visits the tree, starting with the leaves and finally the root (bottom-up)" for child in tree.children: if isinstance(child, Tree): @@ -275,7 +281,7 @@ class Visitor_Recursive(VisitorBase): self._call_userfunc(tree) return tree - def visit_topdown(self,tree): + def visit_topdown(self,tree: Tree) -> Tree: "Visit the tree, starting at the root, and ending at the leaves (top-down)" self._call_userfunc(tree) @@ -286,16 +292,7 @@ class Visitor_Recursive(VisitorBase): return tree -def visit_children_decor(func): - "See Interpreter" - @wraps(func) - def inner(cls, tree): - values = cls.visit_children(tree) - return func(cls, values) - return inner - - -class Interpreter(_Decoratable): +class Interpreter(_Decoratable, ABC, Generic[_T]): """Interpreter walks the tree starting at the root. Visits the tree, starting with the root and finally the leaves (top-down) @@ -307,7 +304,7 @@ class Interpreter(_Decoratable): This allows the user to implement branching and loops. """ - def visit(self, tree): + def visit(self, tree: Tree) -> _T: f = getattr(self, tree.data) wrapper = getattr(f, 'visit_wrapper', None) if wrapper is not None: @@ -315,7 +312,7 @@ class Interpreter(_Decoratable): else: return f(tree) - def visit_children(self, tree): + def visit_children(self, tree: Tree) -> List[_T]: return [self.visit(child) if isinstance(child, Tree) else child for child in tree.children] @@ -326,6 +323,16 @@ class Interpreter(_Decoratable): return self.visit_children(tree) +_InterMethod = Callable[[Type[Interpreter], _T], _R] + +def visit_children_decor(func: _InterMethod) -> _InterMethod: + "See Interpreter" + @wraps(func) + def inner(cls, tree): + values = cls.visit_children(tree) + return func(cls, values) + return inner + # Decorators def _apply_decorator(obj, decorator, **kwargs): @@ -380,7 +387,7 @@ def _vargs_tree(f, data, children, meta): return f(Tree(data, children, meta)) -def v_args(inline=False, meta=False, tree=False, wrapper=None): +def v_args(inline: bool=False, meta: bool=False, tree: bool=False, wrapper: Callable[[_DECORATED], _DECORATED]=None) -> Callable[[_DECORATED], _DECORATED]: """A convenience decorator factory for modifying the behavior of user-supplied visitor methods. By default, callback methods of transformers/visitors accept one argument - a list of the node's children.