diff --git a/lark-stubs/grammar.pyi b/lark-stubs/grammar.pyi index 379d7a9..3a3d806 100644 --- a/lark-stubs/grammar.pyi +++ b/lark-stubs/grammar.pyi @@ -6,4 +6,9 @@ class RuleOptions: expand1: bool priority: int template_source: Optional[str] - empty_indices: Tuple[bool, ...] \ No newline at end of file + empty_indices: Tuple[bool, ...] + + +class Symbol: + name: str + is_term: bool diff --git a/lark-stubs/indenter.pyi b/lark-stubs/indenter.pyi index f0cf8a2..3a7aa97 100644 --- a/lark-stubs/indenter.pyi +++ b/lark-stubs/indenter.pyi @@ -3,9 +3,10 @@ from typing import Tuple, List, Iterator, Optional from abc import ABC, abstractmethod from .lexer import Token +from .lark import PostLex -class Indenter(ABC): +class Indenter(PostLex, ABC): paren_level: Optional[int] indent_level: Optional[List[int]] @@ -15,13 +16,6 @@ class Indenter(ABC): def handle_NL(self, token: Token) -> Iterator[Token]: ... - def process(self, stream: Iterator[Token]) -> Iterator[Token]: - ... - - @property - def always_accept(self) -> Tuple[str]: - ... - @property @abstractmethod def NL_type(self) -> str: diff --git a/lark-stubs/lark.pyi b/lark-stubs/lark.pyi index 9246938..fba567b 100644 --- a/lark-stubs/lark.pyi +++ b/lark-stubs/lark.pyi @@ -65,7 +65,7 @@ class Lark: grammar: Union[Grammar, str, IO[str]], *, start: Union[None, str, List[str]] = "start", - parser: Literal["earley", "lalr", "cyk"] = "auto", + parser: Literal["earley", "lalr", "cyk", "auto"] = "auto", lexer: Union[Literal["auto", "standard", "contextual", "dynamic", "dynamic_complete"], Type[Lexer]] = "auto", transformer: Optional[Transformer] = None, postlex: Optional[PostLex] = None, diff --git a/lark-stubs/load_grammar.pyi b/lark-stubs/load_grammar.pyi index 0521e33..7202fa8 100644 --- a/lark-stubs/load_grammar.pyi +++ b/lark-stubs/load_grammar.pyi @@ -1,8 +1,8 @@ from typing import List, Tuple, Union, Callable, Dict, Optional -from lark import Tree -from lark.grammar import RuleOptions -from lark.exceptions import UnexpectedInput +from .tree import Tree +from .grammar import RuleOptions +from .exceptions import UnexpectedInput class Grammar: diff --git a/lark-stubs/reconstruct.pyi b/lark-stubs/reconstruct.pyi index 2824fa7..9826428 100644 --- a/lark-stubs/reconstruct.pyi +++ b/lark-stubs/reconstruct.pyi @@ -11,8 +11,7 @@ from .lexer import TerminalDef class WriteTokensTransformer(Transformer_InPlace): - def __init__(self, tokens: Dict[str, TerminalDef], Dict[str, Callable[[Symbol], str]] = ...): - ... + def __init__(self, tokens: Dict[str, TerminalDef], term_subs: Dict[str, Callable[[Symbol], str]] = ...): ... class MatchTree(Tree): diff --git a/lark/indenter.py b/lark/indenter.py index 7663be7..7e1263d 100644 --- a/lark/indenter.py +++ b/lark/indenter.py @@ -1,13 +1,14 @@ "Provides Indentation services for languages with indentation similar to Python" from .exceptions import LarkError +from .lark import PostLex from .lexer import Token ###{standalone class DedentError(LarkError): pass -class Indenter: +class Indenter(PostLex): def __init__(self): self.paren_level = None self.indent_level = None diff --git a/lark/lark.py b/lark/lark.py index 4aa9e7e..0b5e388 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -1,4 +1,6 @@ from __future__ import absolute_import + + from lark.exceptions import ConfigurationError, assert_config import sys, os, pickle, hashlib @@ -6,7 +8,7 @@ from io import open import tempfile from warnings import warn -from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger +from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger, ABC, abstractmethod from .load_grammar import load_grammar, FromPackageLoader, Grammar from .tree import Tree from .common import LexerConf, ParserConf @@ -191,6 +193,14 @@ _VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None) _VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest') +class PostLex(ABC): + @abstractmethod + def process(self, stream): + return stream + + always_accept = () + + class Lark(Serialize): """Main interface for the library. diff --git a/lark/tools/standalone.py b/lark/tools/standalone.py index 424dd95..038e181 100644 --- a/lark/tools/standalone.py +++ b/lark/tools/standalone.py @@ -56,7 +56,6 @@ EXTRACT_STANDALONE_FILES = [ 'utils.py', 'tree.py', 'visitors.py', - 'indenter.py', 'grammar.py', 'lexer.py', 'common.py', @@ -65,6 +64,7 @@ EXTRACT_STANDALONE_FILES = [ 'parsers/lalr_analysis.py', 'parser_frontends.py', 'lark.py', + 'indenter.py', ] def extract_sections(lines): diff --git a/lark/utils.py b/lark/utils.py index 023b118..f3bd957 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -12,6 +12,15 @@ logger.addHandler(logging.StreamHandler()) # By default, we should not output any log messages logger.setLevel(logging.CRITICAL) +if sys.version_info[0]>2: + from abc import ABC, abstractmethod +else: + from abc import ABCMeta, abstractmethod + class ABC(object): # Provide Python27 compatibility + __slots__ = () + __metclass__ = ABCMeta + + Py36 = (sys.version_info[:2] >= (3, 6)) NO_VALUE = object()