Browse Source

Continue merging .pyi files

load_grammar, reconstruct, and visitors
gm/2021-09-23T00Z/github.com--lark-parser-lark/1.0b
Chanic Panic 3 years ago
parent
commit
d7b819e9ce
7 changed files with 56 additions and 223 deletions
  1. +0
    -31
      lark-stubs/load_grammar.pyi
  2. +0
    -39
      lark-stubs/reconstruct.pyi
  3. +0
    -108
      lark-stubs/visitors.pyi
  4. +2
    -1
      lark/indenter.py
  5. +14
    -13
      lark/load_grammar.py
  6. +7
    -5
      lark/reconstruct.py
  7. +33
    -26
      lark/visitors.py

+ 0
- 31
lark-stubs/load_grammar.pyi View File

@@ -1,31 +0,0 @@
from typing import List, Tuple, Union, Callable, Dict, Optional

from .tree import Tree
from .grammar import RuleOptions
from .exceptions import UnexpectedInput


class Grammar:
rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]]
term_defs: List[Tuple[str, Tuple[Tree, int]]]
ignore: List[str]


class GrammarBuilder:
global_keep_all_tokens: bool
import_paths: List[Union[str, Callable]]
used_files: Dict[str, str]

def __init__(self, global_keep_all_tokens: bool = False, import_paths: List[Union[str, Callable]] = None, used_files: Dict[str, str]=None) -> None: ...

def load_grammar(self, grammar_text: str, grammar_name: str = ..., mangle: Callable[[str], str] = None) -> None: ...

def do_import(self, dotted_path: Tuple[str, ...], base_path: Optional[str], aliases: Dict[str, str],
base_mangle: Callable[[str], str] = None) -> None: ...

def validate(self) -> None: ...

def build(self) -> Grammar: ...


def find_grammar_errors(text: str, start: str='start') -> List[Tuple[UnexpectedInput, str]]: ...

+ 0
- 39
lark-stubs/reconstruct.pyi View File

@@ -1,39 +0,0 @@
# -*- coding: utf-8 -*-

from typing import List, Dict, Union, Callable, Iterable

from .grammar import Symbol
from .lark import Lark
from .tree import Tree
from .visitors import Transformer_InPlace
from .lexer import TerminalDef


class WriteTokensTransformer(Transformer_InPlace):

def __init__(self, tokens: Dict[str, TerminalDef], term_subs: Dict[str, Callable[[Symbol], str]] = ...): ...


class MatchTree(Tree):
pass


class MakeMatchTree:
name: str
expansion: List[TerminalDef]

def __init__(self, name: str, expansion: List[TerminalDef]):
...

def __call__(self, args: List[Union[str, Tree]]):
...


class Reconstructor:

def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]] = ...):
...

def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]=None,
insert_spaces: bool = True) -> str:
...

+ 0
- 108
lark-stubs/visitors.pyi View File

@@ -1,108 +0,0 @@
# -*- coding: utf-8 -*-

from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union
from abc import ABC
from .tree import Tree

_T = TypeVar('_T')
_R = TypeVar('_R')
_FUNC = Callable[..., _T]
_DECORATED = Union[_FUNC, type]


class Transformer(ABC, Generic[_T]):

def __init__(self, visit_tokens: bool = True) -> None:
...

def transform(self, tree: Tree) -> _T:
...

def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]:
...


class TransformerChain(Generic[_T]):
transformers: Tuple[Transformer[_T], ...]

def __init__(self, *transformers: Transformer[_T]) -> None:
...

def transform(self, tree: Tree) -> _T:
...

def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]:
...


class Transformer_InPlace(Transformer):
pass


class Transformer_NonRecursive(Transformer):
pass


class Transformer_InPlaceRecursive(Transformer):
pass


class VisitorBase:
pass


class Visitor(VisitorBase, ABC, Generic[_T]):

def visit(self, tree: Tree) -> Tree:
...

def visit_topdown(self, tree: Tree) -> Tree:
...


class Visitor_Recursive(VisitorBase):

def visit(self, tree: Tree) -> Tree:
...

def visit_topdown(self, tree: Tree) -> Tree:
...


class Interpreter(ABC, Generic[_T]):

def visit(self, tree: Tree) -> _T:
...

def visit_children(self, tree: Tree) -> List[_T]:
...


_InterMethod = Callable[[Type[Interpreter], _T], _R]


def v_args(
inline: bool = False,
meta: bool = False,
tree: bool = False,
wrapper: Callable = None
) -> Callable[[_DECORATED], _DECORATED]:
...


def visit_children_decor(func: _InterMethod) -> _InterMethod:
...


class Discard(Exception):
pass


# Deprecated
class InlineTransformer:
pass


# Deprecated
def inline_args(obj: _FUNC) -> _FUNC:
...

+ 2
- 1
lark/indenter.py View File

@@ -1,13 +1,14 @@
"Provides Indentation services for languages with indentation similar to Python"

from abc import ABC, abstractmethod
from typing import Tuple, List, Iterator, Optional

from .exceptions import LarkError
from .lark import PostLex
from .lexer import Token

###{standalone
from typing import Tuple, List, Iterator, Optional

class DedentError(LarkError):
pass



+ 14
- 13
lark/load_grammar.py View File

@@ -8,6 +8,7 @@ import pkgutil
from ast import literal_eval
from numbers import Integral
from contextlib import suppress
from typing import List, Tuple, Union, Callable, Dict, Optional

from .utils import bfs, Py36, logger, classify_bool, is_id_continue, is_id_start, bfs_all_unique
from .lexer import Token, TerminalDef, PatternStr, PatternRE
@@ -17,7 +18,7 @@ from .parser_frontends import ParsingFrontend
from .common import LexerConf, ParserConf
from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
from .utils import classify, dedup_list
from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError
from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError, UnexpectedInput

from .tree import Tree, SlottedTree as ST
from .visitors import Transformer, Visitor, v_args, Transformer_InPlace, Transformer_NonRecursive
@@ -552,9 +553,9 @@ def nr_deepcopy_tree(t):

class Grammar:
def __init__(self, rule_defs, term_defs, ignore):
self.term_defs = term_defs
self.rule_defs = rule_defs
self.ignore = ignore
self.term_defs: List[Tuple[str, Tuple[Tree, int]]] = term_defs
self.rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]] = rule_defs
self.ignore: List[str] = ignore

def compile(self, start, terminals_to_keep):
# We change the trees in-place (to support huge grammars)
@@ -874,7 +875,7 @@ def _search_interactive_parser(interactive_parser, predicate):
if predicate(p):
return path, p

def find_grammar_errors(text, start='start'):
def find_grammar_errors(text: str, start: str='start') -> List[Tuple[UnexpectedInput, str]]:
errors = []
def on_error(e):
errors.append((e, _error_repr(e)))
@@ -923,10 +924,10 @@ def _mangle_exp(exp, mangle):


class GrammarBuilder:
def __init__(self, global_keep_all_tokens=False, import_paths=None, used_files=None):
self.global_keep_all_tokens = global_keep_all_tokens
self.import_paths = import_paths or []
self.used_files = used_files or {}
def __init__(self, global_keep_all_tokens: bool=False, import_paths: List[Union[str, Callable]]=None, used_files: Dict[str, str]=None) -> None:
self.global_keep_all_tokens: bool = global_keep_all_tokens
self.import_paths: List[Union[str, Callable]] = import_paths or []
self.used_files: Dict[str, str] = used_files or {}

self._definitions = {}
self._ignore_names = []
@@ -1067,7 +1068,7 @@ class GrammarBuilder:
return name, exp, params, opts


def load_grammar(self, grammar_text, grammar_name="<?>", mangle=None):
def load_grammar(self, grammar_text: str, grammar_name: str="<?>", mangle: Callable[[str], str]=None) -> None:
tree = _parse_grammar(grammar_text, grammar_name)

imports = {}
@@ -1130,7 +1131,7 @@ class GrammarBuilder:
self._definitions = {k: v for k, v in self._definitions.items() if k in _used}


def do_import(self, dotted_path, base_path, aliases, base_mangle=None):
def do_import(self, dotted_path: Tuple[str, ...], base_path: Optional[str], aliases: Dict[str, str], base_mangle: Callable[[str], str]=None) -> None:
assert dotted_path
mangle = _get_mangle('__'.join(dotted_path), aliases, base_mangle)
grammar_path = os.path.join(*dotted_path) + EXT
@@ -1166,7 +1167,7 @@ class GrammarBuilder:
assert False, "Couldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for it" % (dotted_path,)


def validate(self):
def validate(self) -> None:
for name, (params, exp, _options) in self._definitions.items():
for i, p in enumerate(params):
if p in self._definitions:
@@ -1195,7 +1196,7 @@ class GrammarBuilder:
if not set(self._definitions).issuperset(self._ignore_names):
raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(self._ignore_names) - set(self._definitions)))

def build(self):
def build(self) -> Grammar:
self.validate()
rule_defs = []
term_defs = []


+ 7
- 5
lark/reconstruct.py View File

@@ -1,11 +1,13 @@
"""Reconstruct text from a tree, based on Lark grammar"""

from typing import List, Dict, Union, Callable, Iterable
import unicodedata

from .lark import Lark
from .tree import Tree
from .visitors import Transformer_InPlace
from .lexer import Token, PatternStr
from .grammar import Terminal, NonTerminal
from .lexer import Token, PatternStr, TerminalDef
from .grammar import Terminal, NonTerminal, Symbol

from .tree_matcher import TreeMatcher, is_discarded_terminal
from .utils import is_id_continue
@@ -21,7 +23,7 @@ def is_iter_empty(i):
class WriteTokensTransformer(Transformer_InPlace):
"Inserts discarded tokens into their correct place, according to the rules of grammar"

def __init__(self, tokens, term_subs):
def __init__(self, tokens: Dict[str, TerminalDef], term_subs: Dict[str, Callable[[Symbol], str]]) -> None:
self.tokens = tokens
self.term_subs = term_subs

@@ -70,7 +72,7 @@ class Reconstructor(TreeMatcher):
term_subs: a dictionary of [Terminal name as str] to [output text as str]
"""

def __init__(self, parser, term_subs=None):
def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]]=None) -> None:
TreeMatcher.__init__(self, parser)

self.write_tokens = WriteTokensTransformer({t.name:t for t in self.tokens}, term_subs or {})
@@ -87,7 +89,7 @@ class Reconstructor(TreeMatcher):
else:
yield item

def reconstruct(self, tree, postproc=None, insert_spaces=True):
def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]=None, insert_spaces: bool=True) -> str:
x = self._reconstruct(tree)
if postproc:
x = postproc(x)


+ 33
- 26
lark/visitors.py View File

@@ -1,3 +1,4 @@
from abc import ABC
from functools import wraps

from .utils import smart_decorator, combine_alternatives
@@ -7,7 +8,12 @@ from .lexer import Token

###{standalone
from inspect import getmembers, getmro
from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union

_T = TypeVar('_T')
_R = TypeVar('_R')
_FUNC = Callable[..., _T]
_DECORATED = Union[_FUNC, type]

class Discard(Exception):
"""When raising the Discard exception in a transformer callback,
@@ -46,7 +52,7 @@ class _Decoratable:
return cls


class Transformer(_Decoratable):
class Transformer(_Decoratable, ABC, Generic[_T]):
"""Transformers visit each node of the tree, and run the appropriate method on it according to the node's data.

Methods are provided by the user via inheritance, and called according to ``tree.data``.
@@ -74,7 +80,7 @@ class Transformer(_Decoratable):
"""
__visit_tokens__ = True # For backwards compatibility

def __init__(self, visit_tokens=True):
def __init__(self, visit_tokens: bool=True) -> None:
self.__visit_tokens__ = visit_tokens

def _call_userfunc(self, tree, new_children=None):
@@ -125,11 +131,11 @@ class Transformer(_Decoratable):
children = list(self._transform_children(tree.children))
return self._call_userfunc(tree, children)

def transform(self, tree):
def transform(self, tree: Tree) -> _T:
"Transform the given tree, and return the final result"
return self._transform_tree(tree)

def __mul__(self, other):
def __mul__(self, other: 'Transformer[_T]') -> 'TransformerChain[_T]':
"""Chain two transformers together, returning a new transformer.
"""
return TransformerChain(self, other)
@@ -149,16 +155,16 @@ class Transformer(_Decoratable):
return token


class TransformerChain(object):
class TransformerChain(Generic[_T]):
def __init__(self, *transformers):
self.transformers = transformers
self.transformers: Tuple[Transformer[_T], ...] = transformers

def transform(self, tree):
def transform(self, tree: Tree) -> _T:
for t in self.transformers:
tree = t.transform(tree)
return tree

def __mul__(self, other):
def __mul__(self, other: Transformer[_T]) -> 'TransformerChain[_T]':
return TransformerChain(*self.transformers + (other,))


@@ -239,19 +245,19 @@ class VisitorBase:
return cls


class Visitor(VisitorBase):
class Visitor(VisitorBase, ABC, Generic[_T]):
"""Tree visitor, non-recursive (can handle huge trees).

Visiting a node calls its methods (provided by the user via inheritance) according to ``tree.data``
"""

def visit(self, tree):
def visit(self, tree: Tree) -> Tree:
"Visits the tree, starting with the leaves and finally the root (bottom-up)"
for subtree in tree.iter_subtrees():
self._call_userfunc(subtree)
return tree

def visit_topdown(self,tree):
def visit_topdown(self, tree: Tree) -> Tree:
"Visit the tree, starting at the root, and ending at the leaves (top-down)"
for subtree in tree.iter_subtrees_topdown():
self._call_userfunc(subtree)
@@ -266,7 +272,7 @@ class Visitor_Recursive(VisitorBase):
Slightly faster than the non-recursive version.
"""

def visit(self, tree):
def visit(self, tree: Tree) -> Tree:
"Visits the tree, starting with the leaves and finally the root (bottom-up)"
for child in tree.children:
if isinstance(child, Tree):
@@ -275,7 +281,7 @@ class Visitor_Recursive(VisitorBase):
self._call_userfunc(tree)
return tree

def visit_topdown(self,tree):
def visit_topdown(self,tree: Tree) -> Tree:
"Visit the tree, starting at the root, and ending at the leaves (top-down)"
self._call_userfunc(tree)

@@ -286,16 +292,7 @@ class Visitor_Recursive(VisitorBase):
return tree


def visit_children_decor(func):
"See Interpreter"
@wraps(func)
def inner(cls, tree):
values = cls.visit_children(tree)
return func(cls, values)
return inner


class Interpreter(_Decoratable):
class Interpreter(_Decoratable, ABC, Generic[_T]):
"""Interpreter walks the tree starting at the root.

Visits the tree, starting with the root and finally the leaves (top-down)
@@ -307,7 +304,7 @@ class Interpreter(_Decoratable):
This allows the user to implement branching and loops.
"""

def visit(self, tree):
def visit(self, tree: Tree) -> _T:
f = getattr(self, tree.data)
wrapper = getattr(f, 'visit_wrapper', None)
if wrapper is not None:
@@ -315,7 +312,7 @@ class Interpreter(_Decoratable):
else:
return f(tree)

def visit_children(self, tree):
def visit_children(self, tree: Tree) -> List[_T]:
return [self.visit(child) if isinstance(child, Tree) else child
for child in tree.children]

@@ -326,6 +323,16 @@ class Interpreter(_Decoratable):
return self.visit_children(tree)


_InterMethod = Callable[[Type[Interpreter], _T], _R]

def visit_children_decor(func: _InterMethod) -> _InterMethod:
"See Interpreter"
@wraps(func)
def inner(cls, tree):
values = cls.visit_children(tree)
return func(cls, values)
return inner

# Decorators

def _apply_decorator(obj, decorator, **kwargs):
@@ -380,7 +387,7 @@ def _vargs_tree(f, data, children, meta):
return f(Tree(data, children, meta))


def v_args(inline=False, meta=False, tree=False, wrapper=None):
def v_args(inline: bool=False, meta: bool=False, tree: bool=False, wrapper: Callable[[_DECORATED], _DECORATED]=None) -> Callable[[_DECORATED], _DECORATED]:
"""A convenience decorator factory for modifying the behavior of user-supplied visitor methods.

By default, callback methods of transformers/visitors accept one argument - a list of the node's children.


Loading…
Cancel
Save