From 25db16dd73bb0ddd354955e3277fa37489cd2199 Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Thu, 13 Feb 2020 22:34:05 +0800 Subject: [PATCH 1/9] Add stubs for PEP 561. --- .gitignore | 1 + lark/exceptions.pyi | 42 +++++++++++++++++++++++ lark/lark.pyi | 32 +++++++++++++++++ lark/lexer.pyi | 64 ++++++++++++++++++++++++++++++++++ lark/py.typed | 0 lark/reconstruct.pyi | 13 +++++++ lark/tree.pyi | 39 +++++++++++++++++++++ lark/visitors.py | 10 ++++-- lark/visitors.pyi | 82 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 10 files changed, 281 insertions(+), 4 deletions(-) create mode 100644 lark/exceptions.pyi create mode 100644 lark/lark.pyi create mode 100644 lark/lexer.pyi create mode 100644 lark/py.typed create mode 100644 lark/reconstruct.pyi create mode 100644 lark/tree.pyi create mode 100644 lark/visitors.pyi diff --git a/.gitignore b/.gitignore index 710a131..d5fc864 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ tags .idea .ropeproject .cache +.mypy_cache /dist /build diff --git a/lark/exceptions.pyi b/lark/exceptions.pyi new file mode 100644 index 0000000..6c3c4ab --- /dev/null +++ b/lark/exceptions.pyi @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +from typing import Dict, Iterable, Callable +from .tree import Tree + + +class LarkError(Exception): + pass + + +class GrammarError(LarkError): + pass + + +class ParseError(LarkError): + pass + + +class LexError(LarkError): + pass + + +class UnexpectedInput(LarkError): + pos_in_stream: int + + def get_context(self, text: str, span: int = 40): + ... + + def match_examples( + self, + parse_fn: Callable[[str], Tree], + examples: Dict[str, Iterable[str]] + ): + ... + + +class UnexpectedToken(ParseError, UnexpectedInput): + pass + + +class UnexpectedCharacters(LexError, UnexpectedInput): + pass diff --git a/lark/lark.pyi b/lark/lark.pyi new file mode 100644 index 0000000..420cc59 --- /dev/null +++ b/lark/lark.pyi @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +from typing import List, Dict, IO, Callable, Union, Optional, Literal +from .visitors import Transformer +from .lexer import Lexer, Token +from .tree import Tree + +_Start = Union[None, str, List[str]] + + +class Lark: + + def __init__( + self, + grammar: Union[str, IO[str]], + *, + start: _Start = ..., + parser: Literal["earley", "lalr", "cyk"] = ..., + lexer: Optional[Lexer] = ..., + transformer: Optional[Transformer] = ..., + postlex: Optional[Literal["standard", "contextual"]] = ..., + ambiguity: Literal["explicit", "resolve"] = ..., + debug: bool = False, + keep_all_tokens: bool = False, + propagate_positions: bool = False, + maybe_placeholders: bool = False, + lexer_callbacks: Dict[str, Callable[[Token], Token]] + ): + ... + + def parse(self, text: str, start: _Start = None) -> Tree: + ... diff --git a/lark/lexer.pyi b/lark/lexer.pyi new file mode 100644 index 0000000..abe321b --- /dev/null +++ b/lark/lexer.pyi @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +from typing import Tuple, Iterator, Sized +from abc import abstractmethod, ABC + + +class Pattern(ABC): + + @abstractmethod + def to_regexp(self) -> str: + ... + + +class PatternStr(Pattern): + + def to_regexp(self) -> str: + ... + + +class PatternRE(Pattern): + + def to_regexp(self) -> str: + ... + + +class TerminalDef: + name: str + pattern: Pattern + priority: int + + +class Token(str): + type: str + pos_in_stream: int + line: int + column: int + end_line: int + end_column: int + end_pos: int + + +class Lexer(ABC): + + @abstractmethod + def lex(self, stream: Sized) -> Iterator[Token]: + ... + + +class TraditionalLexer(Lexer): + + def build(self) -> None: + ... + + def match(self, stream: str, pos: int) -> Tuple[str, str]: + ... + + def lex(self, stream: Sized) -> Iterator[Token]: + ... + + +class ContextualLexer(Lexer): + + def lex(self, stream: Sized) -> Iterator[Token]: + ... diff --git a/lark/py.typed b/lark/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/lark/reconstruct.pyi b/lark/reconstruct.pyi new file mode 100644 index 0000000..89e51d4 --- /dev/null +++ b/lark/reconstruct.pyi @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +from .lark import Lark +from .tree import Tree + + +class Reconstructor: + + def __init__(self, parser: Lark): + ... + + def reconstruct(self, tree: Tree) -> str: + ... diff --git a/lark/tree.pyi b/lark/tree.pyi new file mode 100644 index 0000000..b794f53 --- /dev/null +++ b/lark/tree.pyi @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +from typing import List, Callable, Iterator, Union, Optional +from .lexer import Token + + +class Tree: + + data: str + children: List[Union[str, Tree]] + meta: Token + + def __init__(self, data: str, children: List[Tree], meta: Optional[Token] = None): + ... + + def pretty(self, indent_str: str = ...) -> str: + ... + + def find_pred(self, pred: Callable[[Tree], bool]) -> Iterator[Tree]: + ... + + def find_data(self, data: str) -> Iterator[Tree]: + ... + + def iter_subtrees(self) -> Iterator[Tree]: + ... + + def iter_subtrees_topdown(self) -> Iterator[Tree]: + ... + + def __eq__(self, other: object) -> bool: + ... + + def __hash__(self) -> int: + ... + + +class SlottedTree(Tree): + pass diff --git a/lark/visitors.py b/lark/visitors.py index 30a2a65..12e296f 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -36,8 +36,12 @@ class _Decoratable: return cls +class _GenericMeta(type): + def __getitem__(self, _): + return self -class Transformer(_Decoratable): + +class Transformer(_Decoratable, metaclass=_GenericMeta): """Visits the tree recursively, starting with the leaves and finally the root (bottom-up) Calls its methods (provided by user via inheritance) according to tree.data @@ -171,7 +175,7 @@ class VisitorBase: return tree -class Visitor(VisitorBase): +class Visitor(VisitorBase, metaclass=_GenericMeta): """Bottom-up visitor, non-recursive Visits the tree, starting with the leaves and finally the root (bottom-up) @@ -224,7 +228,7 @@ def visit_children_decor(func): return inner -class Interpreter(_Decoratable): +class Interpreter(_Decoratable, metaclass=_GenericMeta): """Top-down visitor, recursive Visits the tree, starting with the root and finally the leaves (top-down) diff --git a/lark/visitors.pyi b/lark/visitors.pyi new file mode 100644 index 0000000..ac1367a --- /dev/null +++ b/lark/visitors.pyi @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +from typing import TypeVar, List, Callable, Generic, Type +from abc import ABC +from .tree import Tree + +_T = TypeVar('_T') +_R = TypeVar('_R') +_FUNC = Callable[..., _T] + + +class Transformer(ABC, Generic[_T]): + + def __init__(self, visit_tokens: bool = True): + ... + + def transform(self, tree: Tree) -> _T: + ... + + +class Transformer_InPlace(Transformer): + pass + + +class VisitorBase: + pass + + +class Visitor(VisitorBase, ABC, Generic[_T]): + + def visit(self, tree: Tree) -> Tree: + ... + + def visit_topdown(self, tree: Tree) -> Tree: + ... + + +class Visitor_Recursive(VisitorBase): + + def visit(self, tree: Tree) -> Tree: + ... + + def visit_topdown(self, tree: Tree) -> Tree: + ... + + +class Interpreter(ABC, Generic[_T]): + + def visit(self, tree: Tree) -> _T: + ... + + def visit_children(self, tree: Tree) -> List[_T]: + ... + + +_InterMethod = Callable[[Type[Interpreter], _T], _R] + + +def v_args( + inline: bool = False, + meta: bool = False, + tree: bool = False +) -> Callable[[_FUNC], _FUNC]: + ... + + +def visit_children_decor(func: _InterMethod) -> _InterMethod: + ... + + +class Discard(Exception): + pass + + +# Deprecated +class InlineTransformer: + pass + + +# Deprecated +def inline_args(obj: _FUNC) -> _FUNC: + ... diff --git a/setup.py b/setup.py index 8543fd4..650bc16 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( requires = [], install_requires = [], - package_data = { '': ['*.md', '*.lark'] }, + package_data = { '': ['*.md', '*.lark', '*.pyi'], 'lark': ['py.typed'] }, test_suite = 'tests.__main__', From 0c63ee74fb1e6f29fe7993fb2c9430dd65d43a71 Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Thu, 13 Feb 2020 23:41:22 +0800 Subject: [PATCH 2/9] Add MyPy action. --- .github/workflows/mypy.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/mypy.yml diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 0000000..f398a4c --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,19 @@ +name: Python type check +on: [push, pull_request] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Download submodules + run: git submodule update --init --recursive + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install mypy + - name: Lint with mypy + run: mypy -m lark || true From 3c77de500b890012f69dd60eb51b98d1236cef5b Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Thu, 13 Feb 2020 23:52:15 +0800 Subject: [PATCH 3/9] Fix syntax for Python 2. --- lark/visitors.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lark/visitors.py b/lark/visitors.py index 12e296f..8d88c96 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -35,13 +35,11 @@ class _Decoratable: setattr(cls, name, decorator(value, static=static, **kwargs)) return cls - -class _GenericMeta(type): - def __getitem__(self, _): - return self + def __class_getitem__(cls, _): + return cls -class Transformer(_Decoratable, metaclass=_GenericMeta): +class Transformer(_Decoratable): """Visits the tree recursively, starting with the leaves and finally the root (bottom-up) Calls its methods (provided by user via inheritance) according to tree.data @@ -49,7 +47,6 @@ class Transformer(_Decoratable, metaclass=_GenericMeta): Can be used to implement map or reduce. """ - __visit_tokens__ = True # For backwards compatibility def __init__(self, visit_tokens=True): self.__visit_tokens__ = visit_tokens @@ -174,15 +171,17 @@ class VisitorBase: "Default operation on tree (for override)" return tree + def __class_getitem__(cls, _): + return cls + -class Visitor(VisitorBase, metaclass=_GenericMeta): +class Visitor(VisitorBase): """Bottom-up visitor, non-recursive Visits the tree, starting with the leaves and finally the root (bottom-up) Calls its methods (provided by user via inheritance) according to tree.data """ - def visit(self, tree): for subtree in tree.iter_subtrees(): self._call_userfunc(subtree) @@ -228,7 +227,7 @@ def visit_children_decor(func): return inner -class Interpreter(_Decoratable, metaclass=_GenericMeta): +class Interpreter(_Decoratable): """Top-down visitor, recursive Visits the tree, starting with the root and finally the leaves (top-down) From 34d3ff9a8d7818b8069bb9aeb346718859c4e6af Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Fri, 14 Feb 2020 00:37:59 +0800 Subject: [PATCH 4/9] Use Python 3.8 for MyPy. --- .github/workflows/mypy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index f398a4c..76b244a 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - name: Install dependencies run: | python -m pip install --upgrade pip From 39aa08d2235f398d7ad40abcb859e85216ac21ea Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Sat, 15 Feb 2020 12:08:58 +0800 Subject: [PATCH 5/9] Split out stubs. --- .github/workflows/mypy.yml | 2 +- lark-stubs/__init__.pyi | 10 ++++++++++ {lark => lark-stubs}/exceptions.pyi | 0 {lark => lark-stubs}/lark.pyi | 0 {lark => lark-stubs}/lexer.pyi | 0 {lark => lark-stubs}/reconstruct.pyi | 0 {lark => lark-stubs}/tree.pyi | 0 {lark => lark-stubs}/visitors.pyi | 0 lark/py.typed | 0 setup.py | 6 +++--- 10 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 lark-stubs/__init__.pyi rename {lark => lark-stubs}/exceptions.pyi (100%) rename {lark => lark-stubs}/lark.pyi (100%) rename {lark => lark-stubs}/lexer.pyi (100%) rename {lark => lark-stubs}/reconstruct.pyi (100%) rename {lark => lark-stubs}/tree.pyi (100%) rename {lark => lark-stubs}/visitors.pyi (100%) delete mode 100644 lark/py.typed diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 76b244a..f1deb42 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -16,4 +16,4 @@ jobs: python -m pip install --upgrade pip pip install mypy - name: Lint with mypy - run: mypy -m lark || true + run: mypy -m lark-stubs || true diff --git a/lark-stubs/__init__.pyi b/lark-stubs/__init__.pyi new file mode 100644 index 0000000..1226e74 --- /dev/null +++ b/lark-stubs/__init__.pyi @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +from .tree import Tree +from .visitors import Transformer, Visitor, v_args, Discard +from .visitors import InlineTransformer, inline_args # XXX Deprecated +from .exceptions import ParseError, LexError, GrammarError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters +from .lexer import Token +from .lark import Lark + +__version__: str diff --git a/lark/exceptions.pyi b/lark-stubs/exceptions.pyi similarity index 100% rename from lark/exceptions.pyi rename to lark-stubs/exceptions.pyi diff --git a/lark/lark.pyi b/lark-stubs/lark.pyi similarity index 100% rename from lark/lark.pyi rename to lark-stubs/lark.pyi diff --git a/lark/lexer.pyi b/lark-stubs/lexer.pyi similarity index 100% rename from lark/lexer.pyi rename to lark-stubs/lexer.pyi diff --git a/lark/reconstruct.pyi b/lark-stubs/reconstruct.pyi similarity index 100% rename from lark/reconstruct.pyi rename to lark-stubs/reconstruct.pyi diff --git a/lark/tree.pyi b/lark-stubs/tree.pyi similarity index 100% rename from lark/tree.pyi rename to lark-stubs/tree.pyi diff --git a/lark/visitors.pyi b/lark-stubs/visitors.pyi similarity index 100% rename from lark/visitors.pyi rename to lark-stubs/visitors.pyi diff --git a/lark/py.typed b/lark/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index 650bc16..b7cc052 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,17 @@ import re -from setuptools import setup +from setuptools import find_packages, setup __version__ ,= re.findall('__version__ = "(.*)"', open('lark/__init__.py').read()) setup( name = "lark-parser", version = __version__, - packages = ['lark', 'lark.parsers', 'lark.tools', 'lark.grammars'], + packages = ['lark', 'lark.parsers', 'lark.tools', 'lark.grammars', 'lark-stubs'], requires = [], install_requires = [], - package_data = { '': ['*.md', '*.lark', '*.pyi'], 'lark': ['py.typed'] }, + package_data = {'': ['*.md', '*.lark'], 'lark-stubs': ['*.pyi']}, test_suite = 'tests.__main__', From 42fa4ed1afc64a140965d4f0e409de9fe20dda04 Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Sat, 15 Feb 2020 18:12:32 +0800 Subject: [PATCH 6/9] Use wildcard import syntax in __init__.pyi. --- lark-stubs/__init__.pyi | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lark-stubs/__init__.pyi b/lark-stubs/__init__.pyi index 1226e74..bb2d018 100644 --- a/lark-stubs/__init__.pyi +++ b/lark-stubs/__init__.pyi @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- -from .tree import Tree -from .visitors import Transformer, Visitor, v_args, Discard -from .visitors import InlineTransformer, inline_args # XXX Deprecated -from .exceptions import ParseError, LexError, GrammarError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters -from .lexer import Token -from .lark import Lark +from .tree import * +from .visitors import * +from .exceptions import * +from .lexer import * +from .lark import * __version__: str From 97afea24d1894ab8ea2ccdc60573ac2a3627070a Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Sun, 16 Feb 2020 09:50:23 +0800 Subject: [PATCH 7/9] Make LarkError public. --- lark/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lark/__init__.py b/lark/__init__.py index 1a00c9d..643da8c 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,7 +1,8 @@ from .tree import Tree from .visitors import Transformer, Visitor, v_args, Discard from .visitors import InlineTransformer, inline_args # XXX Deprecated -from .exceptions import ParseError, LexError, GrammarError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters +from .exceptions import (ParseError, LexError, GrammarError, UnexpectedToken, + UnexpectedInput, UnexpectedCharacters, LarkError) from .lexer import Token from .lark import Lark From 803b1fe79e703d295107a474a9fb0671060f6af7 Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Sun, 16 Feb 2020 14:09:53 +0800 Subject: [PATCH 8/9] Apply changes and extend more names. --- lark-stubs/__init__.pyi | 2 +- lark-stubs/exceptions.pyi | 7 +- lark-stubs/indenter.pyi | 53 +++++++++++++ lark-stubs/lark.pyi | 56 ++++++++++++-- lark-stubs/lexer.pyi | 152 +++++++++++++++++++++++++++++++++++-- lark-stubs/reconstruct.pyi | 24 ++++++ lark-stubs/tree.pyi | 45 +++++++++-- lark-stubs/visitors.pyi | 18 ++++- lark/visitors.py | 1 + 9 files changed, 335 insertions(+), 23 deletions(-) create mode 100644 lark-stubs/indenter.pyi diff --git a/lark-stubs/__init__.pyi b/lark-stubs/__init__.pyi index bb2d018..7545f2d 100644 --- a/lark-stubs/__init__.pyi +++ b/lark-stubs/__init__.pyi @@ -6,4 +6,4 @@ from .exceptions import * from .lexer import * from .lark import * -__version__: str +__version__: str = ... diff --git a/lark-stubs/exceptions.pyi b/lark-stubs/exceptions.pyi index 6c3c4ab..9bdb85d 100644 --- a/lark-stubs/exceptions.pyi +++ b/lark-stubs/exceptions.pyi @@ -23,7 +23,7 @@ class LexError(LarkError): class UnexpectedInput(LarkError): pos_in_stream: int - def get_context(self, text: str, span: int = 40): + def get_context(self, text: str, span: int = ...): ... def match_examples( @@ -39,4 +39,9 @@ class UnexpectedToken(ParseError, UnexpectedInput): class UnexpectedCharacters(LexError, UnexpectedInput): + line: int + column: int + + +class VisitError(LarkError): pass diff --git a/lark-stubs/indenter.pyi b/lark-stubs/indenter.pyi new file mode 100644 index 0000000..11a9058 --- /dev/null +++ b/lark-stubs/indenter.pyi @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +from typing import Tuple, List, Iterator, Optional +from abc import ABC, abstractmethod +from .lexer import Token + + +class Indenter(ABC): + paren_level: Optional[int] + indent_level: Optional[List[int]] + + def __init__(self): + ... + + def handle_NL(self, token: Token) -> Iterator[Token]: + ... + + def process(self, stream: Iterator[Token]) -> Iterator[Token]: + ... + + @property + def always_accept(self) -> Tuple[str]: + ... + + @property + @abstractmethod + def NL_type(self) -> str: + ... + + @property + @abstractmethod + def OPEN_PAREN_types(self) -> List[str]: + ... + + @property + @abstractmethod + def CLOSE_PAREN_types(self) -> List[str]: + ... + + @property + @abstractmethod + def INDENT_type(self) -> str: + ... + + @property + @abstractmethod + def DEDENT_type(self) -> str: + ... + + @property + @abstractmethod + def tab_len(self) -> int: + ... diff --git a/lark-stubs/lark.pyi b/lark-stubs/lark.pyi index 420cc59..b3aa36d 100644 --- a/lark-stubs/lark.pyi +++ b/lark-stubs/lark.pyi @@ -1,25 +1,57 @@ # -*- coding: utf-8 -*- -from typing import List, Dict, IO, Callable, Union, Optional, Literal +from typing import ( + TypeVar, Type, List, Dict, IO, Iterator, Callable, Union, Optional, + Literal, Protocol, +) from .visitors import Transformer -from .lexer import Lexer, Token +from .lexer import Token, Lexer, TerminalDef from .tree import Tree +_T = TypeVar('_T') _Start = Union[None, str, List[str]] +_Parser = Literal["earley", "lalr", "cyk"] +_Lexer = Union[Literal["auto", "standard", "contextual", "dynamic", "dynamic_complete"], Lexer] +_Ambiguity = Literal["explicit", "resolve"] + + +class PostLex(Protocol): + + def process(self, stream: Iterator[Token]) -> Iterator[Token]: + ... + + +class LarkOptions: + start: _Start + parser: _Parser + lexer: _Lexer + transformer: Optional[Transformer] + postlex: Optional[PostLex] + ambiguity: _Ambiguity + debug: bool + keep_all_tokens: bool + propagate_positions: bool + maybe_placeholders: bool + lexer_callbacks: Dict[str, Callable[[Token], Token]] + cache_grammar: bool class Lark: + source: str + options: LarkOptions + lexer: Lexer + terminals: List[TerminalDef] def __init__( self, grammar: Union[str, IO[str]], *, start: _Start = ..., - parser: Literal["earley", "lalr", "cyk"] = ..., - lexer: Optional[Lexer] = ..., - transformer: Optional[Transformer] = ..., - postlex: Optional[Literal["standard", "contextual"]] = ..., - ambiguity: Literal["explicit", "resolve"] = ..., + parser: _Parser = ..., + lexer: _Lexer = ..., + transformer: Optional[Transformer] = None, + postlex: Optional[PostLex] = None, + ambiguity: _Ambiguity = ..., debug: bool = False, keep_all_tokens: bool = False, propagate_positions: bool = False, @@ -30,3 +62,13 @@ class Lark: def parse(self, text: str, start: _Start = None) -> Tree: ... + + @classmethod + def open(cls: Type[_T], grammar_filename: str, rel_to: Optional[str] = None, **options) -> _T: + ... + + def lex(self, text: str) -> Iterator[Token]: + ... + + def get_terminal(self, name: str) -> TerminalDef: + ... diff --git a/lark-stubs/lexer.pyi b/lark-stubs/lexer.pyi index abe321b..171f6ac 100644 --- a/lark-stubs/lexer.pyi +++ b/lark-stubs/lexer.pyi @@ -1,64 +1,202 @@ # -*- coding: utf-8 -*- -from typing import Tuple, Iterator, Sized +from typing import ( + TypeVar, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, + Pattern as REPattern, +) from abc import abstractmethod, ABC +_T = TypeVar('_T') +_MRes = List[Tuple[REPattern, Dict[int, str]]] + class Pattern(ABC): + value: str + flags: Collection[str] + + def __init__(self, value: str, flags: Collection[str] = ...): + ... + + @property + @abstractmethod + def type(self) -> str: + ... @abstractmethod def to_regexp(self) -> str: ... + @property + @abstractmethod + def min_width(self) -> int: + ... + + @property + @abstractmethod + def max_width(self) -> int: + ... + class PatternStr(Pattern): + type: str = ... def to_regexp(self) -> str: ... + @property + def min_width(self) -> int: + ... + + @property + def max_width(self) -> int: + ... + class PatternRE(Pattern): + type: str = ... def to_regexp(self) -> str: ... + @property + def min_width(self) -> int: + ... + + @property + def max_width(self) -> int: + ... + class TerminalDef: name: str pattern: Pattern priority: int + def __init__(self, name: str, pattern: Pattern, priority: int = ...): + ... + class Token(str): type: str pos_in_stream: int + value: str line: int column: int end_line: int end_column: int end_pos: int + def update(self, type_: Optional[str] = None, value: Optional[str] = None) -> Token: + ... -class Lexer(ABC): + @classmethod + def new_borrow_pos(cls: Type[_T], type_: str, value: str, borrow_t: Token) -> _T: + ... - @abstractmethod - def lex(self, stream: Sized) -> Iterator[Token]: + +_Callback = Callable[[Token], Token] + + +def build_mres(terminals: Collection[TerminalDef], match_whole: bool = False) -> _MRes: + ... + + +class UnlessCallback: + mres: _MRes + + def __init__(self, mres: _MRes): + ... + + def __call__(self, t: Token) -> Token: + ... + + +class CallChain: + callback1: _Callback + callback2: _Callback + cond: Callable[[Token], bool] + + def __init__( + self, + callback1: _Callback, + callback2: _Callback, + cond: Callable[[Token], bool] + ): + ... + + +class LineCounter: + newline_char: str + char_pos: int + line: int + column: int + line_start_pos: int + + def __init__(self): + ... + + def feed(self, token: str, test_newline: bool = True): + ... + + +class _Lex: + lexer: TraditionalLexer + + def __init__(self, lexer: TraditionalLexer, state: Optional[str] = None): ... + def lex( + self, + stream: str, + newline_types: Collection[str], + ignore_types: Collection[str] + ) -> Iterator[Token]: + ... + + +class Lexer(ABC): + lex: Callable[..., Iterator[Token]] + class TraditionalLexer(Lexer): + terminals: Collection[TerminalDef] + ignore_types: List[str] + newline_types: List[str] + user_callbacks: Dict[str, _Callback] + callback: Dict[str, _Callback] + mres: _MRes + + def __init__( + self, + terminals: Collection[TerminalDef], + ignore: Collection[str] = ..., + user_callbacks: Dict[str, _Callback] = ... + ): + ... def build(self) -> None: ... - def match(self, stream: str, pos: int) -> Tuple[str, str]: + def match(self, stream: str, pos: int) -> Optional[Tuple[str, str]]: ... - def lex(self, stream: Sized) -> Iterator[Token]: + def lex(self, stream: str) -> Iterator[Token]: ... class ContextualLexer(Lexer): + lexers: Dict[str, TraditionalLexer] + root_lexer: TraditionalLexer + + def __init__( + self, + terminals: Collection[TerminalDef], + states: Dict[str, Collection[str]], + ignore: Collection[str] = ..., + always_accept: Collection[str] = ..., + user_callbacks: Dict[str, _Callback] = ... + ): + ... - def lex(self, stream: Sized) -> Iterator[Token]: + def lex(self, stream: str, get_parser_state: Callable[[], str]) -> Iterator[Token]: ... diff --git a/lark-stubs/reconstruct.pyi b/lark-stubs/reconstruct.pyi index 89e51d4..8ac6c14 100644 --- a/lark-stubs/reconstruct.pyi +++ b/lark-stubs/reconstruct.pyi @@ -1,7 +1,31 @@ # -*- coding: utf-8 -*- +from typing import List, Dict, Union from .lark import Lark from .tree import Tree +from .visitors import Transformer_InPlace +from .lexer import TerminalDef + + +class WriteTokensTransformer(Transformer_InPlace): + + def __init__(self, tokens: Dict[str, TerminalDef], term_subs): + ... + + +class MatchTree(Tree): + pass + + +class MakeMatchTree: + name: str + expansion: List[TerminalDef] + + def __init__(self, name: str, expansion: List[TerminalDef]): + ... + + def __call__(self, args: List[Union[str, Tree]]): + ... class Reconstructor: diff --git a/lark-stubs/tree.pyi b/lark-stubs/tree.pyi index b794f53..a24ab35 100644 --- a/lark-stubs/tree.pyi +++ b/lark-stubs/tree.pyi @@ -1,16 +1,31 @@ # -*- coding: utf-8 -*- -from typing import List, Callable, Iterator, Union, Optional -from .lexer import Token +from typing import List, Callable, Iterator, Union, Optional, Literal +from .lexer import TerminalDef +class Meta: + empty: bool + line: int + column: int + start_pos: int + end_line: int + end_column: int + end_pos: int + orig_expansion: List[TerminalDef] + match_tree: bool -class Tree: +class Tree: data: str children: List[Union[str, Tree]] - meta: Token + meta: Meta - def __init__(self, data: str, children: List[Tree], meta: Optional[Token] = None): + def __init__( + self, + data: str, + children: List[Union[str, Tree]], + meta: Optional[Meta] = None + ): ... def pretty(self, indent_str: str = ...) -> str: @@ -22,13 +37,22 @@ class Tree: def find_data(self, data: str) -> Iterator[Tree]: ... + def expand_kids_by_index(self, *indices: int) -> None: + ... + + def scan_values(self, pred: Callable[[Union[str, Tree]], bool]): + ... + def iter_subtrees(self) -> Iterator[Tree]: ... def iter_subtrees_topdown(self) -> Iterator[Tree]: ... - def __eq__(self, other: object) -> bool: + def copy(self) -> Tree: + ... + + def set(self, data: str, children: List[Union[str, Tree]]) -> None: ... def __hash__(self) -> int: @@ -37,3 +61,12 @@ class Tree: class SlottedTree(Tree): pass + + +def pydot__tree_to_png( + tree: Tree, + filename: str, + rankdir: Literal["TB", "LR", "BT", "RL"] = ..., + **kwargs +) -> None: + ... diff --git a/lark-stubs/visitors.pyi b/lark-stubs/visitors.pyi index ac1367a..2aafd2e 100644 --- a/lark-stubs/visitors.pyi +++ b/lark-stubs/visitors.pyi @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from typing import TypeVar, List, Callable, Generic, Type +from typing import TypeVar, Tuple, List, Callable, Generic, Type from abc import ABC from .tree import Tree @@ -17,6 +17,22 @@ class Transformer(ABC, Generic[_T]): def transform(self, tree: Tree) -> _T: ... + def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]: + ... + + +class TransformerChain(Generic[_T]): + transformers: Tuple[Transformer[_T], ...] + + def __init__(self, *transformers: Transformer[_T]): + ... + + def transform(self, tree: Tree) -> _T: + ... + + def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]: + ... + class Transformer_InPlace(Transformer): pass diff --git a/lark/visitors.py b/lark/visitors.py index 8d88c96..3e3612f 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -48,6 +48,7 @@ class Transformer(_Decoratable): Can be used to implement map or reduce. """ __visit_tokens__ = True # For backwards compatibility + def __init__(self, visit_tokens=True): self.__visit_tokens__ = visit_tokens From d0f9f715f3f89edb3dd7fab90a4c6864098d6f7b Mon Sep 17 00:00:00 2001 From: KmolYuan Date: Mon, 17 Feb 2020 11:05:45 +0800 Subject: [PATCH 9/9] Remove non-public classes in lexer.pyi. --- lark-stubs/lexer.pyi | 60 +------------------------------------------- 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/lark-stubs/lexer.pyi b/lark-stubs/lexer.pyi index 171f6ac..12f4e0a 100644 --- a/lark-stubs/lexer.pyi +++ b/lark-stubs/lexer.pyi @@ -7,7 +7,6 @@ from typing import ( from abc import abstractmethod, ABC _T = TypeVar('_T') -_MRes = List[Tuple[REPattern, Dict[int, str]]] class Pattern(ABC): @@ -97,63 +96,6 @@ class Token(str): _Callback = Callable[[Token], Token] -def build_mres(terminals: Collection[TerminalDef], match_whole: bool = False) -> _MRes: - ... - - -class UnlessCallback: - mres: _MRes - - def __init__(self, mres: _MRes): - ... - - def __call__(self, t: Token) -> Token: - ... - - -class CallChain: - callback1: _Callback - callback2: _Callback - cond: Callable[[Token], bool] - - def __init__( - self, - callback1: _Callback, - callback2: _Callback, - cond: Callable[[Token], bool] - ): - ... - - -class LineCounter: - newline_char: str - char_pos: int - line: int - column: int - line_start_pos: int - - def __init__(self): - ... - - def feed(self, token: str, test_newline: bool = True): - ... - - -class _Lex: - lexer: TraditionalLexer - - def __init__(self, lexer: TraditionalLexer, state: Optional[str] = None): - ... - - def lex( - self, - stream: str, - newline_types: Collection[str], - ignore_types: Collection[str] - ) -> Iterator[Token]: - ... - - class Lexer(ABC): lex: Callable[..., Iterator[Token]] @@ -164,7 +106,7 @@ class TraditionalLexer(Lexer): newline_types: List[str] user_callbacks: Dict[str, _Callback] callback: Dict[str, _Callback] - mres: _MRes + mres: List[Tuple[REPattern, Dict[int, str]]] def __init__( self,