More updates to v1.0remotes/origin/gm/2021-09-23T00Z/github.com--lark-parser-lark/master
@@ -1,14 +1,14 @@ | |||
from types import ModuleType | |||
from copy import deepcopy | |||
from types import ModuleType | |||
from typing import Callable, Collection, Dict, Optional, TYPE_CHECKING | |||
if TYPE_CHECKING: | |||
from .lark import PostLex | |||
from .utils import Serialize | |||
from .lexer import TerminalDef, Token | |||
###{standalone | |||
from typing import Any, Callable, Collection, Dict, Optional, TYPE_CHECKING | |||
if TYPE_CHECKING: | |||
from .lark import PostLex | |||
_Callback = Callable[[Token], Token] | |||
@@ -1,15 +1,13 @@ | |||
from .utils import logger, NO_VALUE | |||
###{standalone | |||
from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, TYPE_CHECKING | |||
from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, Collection, TYPE_CHECKING | |||
if TYPE_CHECKING: | |||
from .lexer import Token | |||
from .parsers.lalr_interactive_parser import InteractiveParser | |||
from .tree import Tree | |||
###{standalone | |||
class LarkError(Exception): | |||
pass | |||
@@ -18,7 +16,7 @@ class ConfigurationError(LarkError, ValueError): | |||
pass | |||
def assert_config(value, options, msg='Got %r, expected one of %s'): | |||
def assert_config(value, options: Collection, msg='Got %r, expected one of %s'): | |||
if value not in options: | |||
raise ConfigurationError(msg % (value, options)) | |||
@@ -1,10 +1,9 @@ | |||
from typing import Optional, Tuple, ClassVar | |||
from .utils import Serialize | |||
###{standalone | |||
from typing import Optional, Tuple, ClassVar | |||
class Symbol(Serialize): | |||
__slots__ = ('name',) | |||
@@ -10,8 +10,8 @@ DEC_NUMBER: /0|[1-9][\d_]*/i | |||
HEX_NUMBER.2: /0x[\da-f]*/i | |||
OCT_NUMBER.2: /0o[0-7]*/i | |||
BIN_NUMBER.2 : /0b[0-1]*/i | |||
FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)(e[-+]?\d+)?|\d+(e[-+]?\d+))/i | |||
IMAG_NUMBER.2: /\d+j/i | FLOAT_NUMBER "j"i | |||
FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)([Ee][-+]?\d+)?|\d+([Ee][-+]?\d+))/ | |||
IMAG_NUMBER.2: /\d+[Jj]/ | FLOAT_NUMBER /[Jj]/ | |||
// Comma-separated list (with an optional trailing comma) | |||
@@ -1,19 +1,18 @@ | |||
"Provides Indentation services for languages with indentation similar to Python" | |||
from abc import ABC, abstractmethod | |||
from typing import List, Iterator | |||
from .exceptions import LarkError | |||
from .lark import PostLex | |||
from .lexer import Token | |||
###{standalone | |||
from typing import Tuple, List, Iterator, Optional | |||
class DedentError(LarkError): | |||
pass | |||
class Indenter(PostLex, ABC): | |||
paren_level: int | |||
indent_level: List[int] | |||
@@ -75,31 +74,31 @@ class Indenter(PostLex, ABC): | |||
@property | |||
@abstractmethod | |||
def NL_type(self) -> str: | |||
... | |||
raise NotImplementedError() | |||
@property | |||
@abstractmethod | |||
def OPEN_PAREN_types(self) -> List[str]: | |||
... | |||
raise NotImplementedError() | |||
@property | |||
@abstractmethod | |||
def CLOSE_PAREN_types(self) -> List[str]: | |||
... | |||
raise NotImplementedError() | |||
@property | |||
@abstractmethod | |||
def INDENT_type(self) -> str: | |||
... | |||
raise NotImplementedError() | |||
@property | |||
@abstractmethod | |||
def DEDENT_type(self) -> str: | |||
... | |||
raise NotImplementedError() | |||
@property | |||
@abstractmethod | |||
def tab_len(self) -> int: | |||
... | |||
raise NotImplementedError() | |||
###} |
@@ -1,7 +1,18 @@ | |||
from abc import ABC, abstractmethod | |||
import sys, os, pickle, hashlib | |||
import tempfile | |||
from typing import ( | |||
TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, | |||
Tuple, Iterable, IO, Any, TYPE_CHECKING | |||
) | |||
if TYPE_CHECKING: | |||
from .parsers.lalr_interactive_parser import InteractiveParser | |||
from .visitors import Transformer | |||
if sys.version_info >= (3, 8): | |||
from typing import Literal | |||
else: | |||
from typing_extensions import Literal | |||
from .exceptions import ConfigurationError, assert_config, UnexpectedInput | |||
from .utils import Serialize, SerializeMemoizer, FS, isascii, logger | |||
from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource | |||
@@ -21,18 +32,7 @@ except ImportError: | |||
###{standalone | |||
from typing import ( | |||
TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, | |||
Tuple, Iterable, IO, Any, TYPE_CHECKING | |||
) | |||
if TYPE_CHECKING: | |||
from .parsers.lalr_interactive_parser import InteractiveParser | |||
from .visitors import Transformer | |||
if sys.version_info >= (3, 8): | |||
from typing import Literal | |||
else: | |||
from typing_extensions import Literal | |||
class PostLex(ABC): | |||
@abstractmethod | |||
@@ -3,6 +3,13 @@ | |||
from abc import abstractmethod, ABC | |||
import re | |||
from contextlib import suppress | |||
from typing import ( | |||
TypeVar, Type, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, | |||
Pattern as REPattern, ClassVar, TYPE_CHECKING | |||
) | |||
from types import ModuleType | |||
if TYPE_CHECKING: | |||
from .common import LexerConf | |||
from .utils import classify, get_regexp_width, Serialize | |||
from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken | |||
@@ -10,14 +17,6 @@ from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken | |||
###{standalone | |||
from copy import copy | |||
from types import ModuleType | |||
from typing import ( | |||
TypeVar, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, | |||
Pattern as REPattern, ClassVar, TYPE_CHECKING | |||
) | |||
if TYPE_CHECKING: | |||
from .common import LexerConf | |||
class Pattern(Serialize, ABC): | |||
@@ -218,7 +217,7 @@ class LineCounter: | |||
return self.char_pos == other.char_pos and self.newline_char == other.newline_char | |||
def feed(self, token, test_newline=True): | |||
def feed(self, token: Token, test_newline=True): | |||
"""Consume a token and calculate the new line & column. | |||
As an optional optimization, set test_newline=False if token doesn't contain a newline. | |||
@@ -320,7 +319,7 @@ class Scanner: | |||
return m.group(0), type_from_index[m.lastindex] | |||
def _regexp_has_newline(r): | |||
def _regexp_has_newline(r: str): | |||
r"""Expressions that may indicate newlines in a regexp: | |||
- newlines (\n) | |||
- escaped newline (\\n) | |||
@@ -359,7 +358,7 @@ class Lexer(ABC): | |||
""" | |||
@abstractmethod | |||
def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]: | |||
... | |||
return NotImplemented | |||
def make_lexer_state(self, text): | |||
line_ctr = LineCounter(b'\n' if isinstance(text, bytes) else '\n') | |||
@@ -1,3 +1,5 @@ | |||
from typing import List | |||
from .exceptions import GrammarError, ConfigurationError | |||
from .lexer import Token | |||
from .tree import Tree | |||
@@ -151,7 +153,7 @@ def _should_expand(sym): | |||
return not sym.is_term and sym.name.startswith('_') | |||
def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices): | |||
def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices: List[bool]): | |||
# Prepare empty_indices as: How many Nones to insert at each index? | |||
if _empty_indices: | |||
assert _empty_indices.count(False) == len(expansion) | |||
@@ -32,20 +32,13 @@ class MakeParsingFrontend: | |||
self.parser_type = parser_type | |||
self.lexer_type = lexer_type | |||
def __call__(self, lexer_conf, parser_conf, options): | |||
assert isinstance(lexer_conf, LexerConf) | |||
assert isinstance(parser_conf, ParserConf) | |||
parser_conf.parser_type = self.parser_type | |||
lexer_conf.lexer_type = self.lexer_type | |||
return ParsingFrontend(lexer_conf, parser_conf, options) | |||
def deserialize(self, data, memo, lexer_conf, callbacks, options): | |||
parser_conf = ParserConf.deserialize(data['parser_conf'], memo) | |||
parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug) | |||
parser_conf.callbacks = callbacks | |||
return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser) | |||
# ... Continued later in the module | |||
class ParsingFrontend(Serialize): | |||
@@ -237,3 +230,12 @@ class CYK_FrontEnd: | |||
def _apply_callback(self, tree): | |||
return self.callbacks[tree.rule](tree.children) | |||
class MakeParsingFrontend(MakeParsingFrontend): | |||
def __call__(self, lexer_conf, parser_conf, options): | |||
assert isinstance(lexer_conf, LexerConf) | |||
assert isinstance(parser_conf, ParserConf) | |||
parser_conf.parser_type = self.parser_type | |||
lexer_conf.lexer_type = self.lexer_type | |||
return ParsingFrontend(lexer_conf, parser_conf, options) |
@@ -25,6 +25,13 @@ | |||
# | |||
from abc import ABC, abstractmethod | |||
from collections.abc import Sequence | |||
from types import ModuleType | |||
from typing import ( | |||
TypeVar, Generic, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, | |||
Union, Iterable, IO, TYPE_CHECKING, | |||
Pattern as REPattern, ClassVar, Set, | |||
) | |||
###} | |||
import sys | |||
@@ -1,3 +1,4 @@ | |||
try: | |||
from future_builtins import filter # type: ignore | |||
except ImportError: | |||
@@ -6,10 +7,7 @@ except ImportError: | |||
import sys | |||
from copy import deepcopy | |||
###{standalone | |||
from collections import OrderedDict | |||
from typing import List, Callable, Iterator, Union, Optional, Any, TYPE_CHECKING | |||
from typing import List, Callable, Iterator, Union, Optional, TYPE_CHECKING | |||
if TYPE_CHECKING: | |||
from .lexer import TerminalDef | |||
@@ -18,6 +16,9 @@ if TYPE_CHECKING: | |||
else: | |||
from typing_extensions import Literal | |||
###{standalone | |||
from collections import OrderedDict | |||
class Meta: | |||
empty: bool | |||
@@ -241,7 +241,7 @@ def combine_alternatives(lists): | |||
try: | |||
import atomicwrites | |||
except ImportError: | |||
atomicwrites = None | |||
atomicwrites = None # type: ignore | |||
class FS: | |||
exists = os.path.exists | |||
@@ -1,3 +1,4 @@ | |||
from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional | |||
from abc import ABC | |||
from functools import wraps | |||
@@ -8,7 +9,6 @@ from .lexer import Token | |||
###{standalone | |||
from inspect import getmembers, getmro | |||
from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional | |||
_T = TypeVar('_T') | |||
_R = TypeVar('_R') | |||
@@ -24,7 +24,7 @@ class TestStandalone(TestCase): | |||
standalone.gen_standalone(Lark(grammar, parser='lalr'), out=code_buf, compress=compress) | |||
code = code_buf.getvalue() | |||
context = {'__doc__': None} | |||
context = {'__doc__': None, '__name__': 'test_standalone'} | |||
exec(code, context) | |||
return context | |||