More updates to v1.0remotes/origin/gm/2021-09-23T00Z/github.com--lark-parser-lark/master
@@ -1,14 +1,14 @@ | |||||
from types import ModuleType | |||||
from copy import deepcopy | from copy import deepcopy | ||||
from types import ModuleType | |||||
from typing import Callable, Collection, Dict, Optional, TYPE_CHECKING | |||||
if TYPE_CHECKING: | |||||
from .lark import PostLex | |||||
from .utils import Serialize | from .utils import Serialize | ||||
from .lexer import TerminalDef, Token | from .lexer import TerminalDef, Token | ||||
###{standalone | ###{standalone | ||||
from typing import Any, Callable, Collection, Dict, Optional, TYPE_CHECKING | |||||
if TYPE_CHECKING: | |||||
from .lark import PostLex | |||||
_Callback = Callable[[Token], Token] | _Callback = Callable[[Token], Token] | ||||
@@ -1,15 +1,13 @@ | |||||
from .utils import logger, NO_VALUE | from .utils import logger, NO_VALUE | ||||
###{standalone | |||||
from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, TYPE_CHECKING | |||||
from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, Collection, TYPE_CHECKING | |||||
if TYPE_CHECKING: | if TYPE_CHECKING: | ||||
from .lexer import Token | from .lexer import Token | ||||
from .parsers.lalr_interactive_parser import InteractiveParser | from .parsers.lalr_interactive_parser import InteractiveParser | ||||
from .tree import Tree | from .tree import Tree | ||||
###{standalone | |||||
class LarkError(Exception): | class LarkError(Exception): | ||||
pass | pass | ||||
@@ -18,7 +16,7 @@ class ConfigurationError(LarkError, ValueError): | |||||
pass | pass | ||||
def assert_config(value, options, msg='Got %r, expected one of %s'): | |||||
def assert_config(value, options: Collection, msg='Got %r, expected one of %s'): | |||||
if value not in options: | if value not in options: | ||||
raise ConfigurationError(msg % (value, options)) | raise ConfigurationError(msg % (value, options)) | ||||
@@ -1,10 +1,9 @@ | |||||
from typing import Optional, Tuple, ClassVar | |||||
from .utils import Serialize | from .utils import Serialize | ||||
###{standalone | ###{standalone | ||||
from typing import Optional, Tuple, ClassVar | |||||
class Symbol(Serialize): | class Symbol(Serialize): | ||||
__slots__ = ('name',) | __slots__ = ('name',) | ||||
@@ -10,8 +10,8 @@ DEC_NUMBER: /0|[1-9][\d_]*/i | |||||
HEX_NUMBER.2: /0x[\da-f]*/i | HEX_NUMBER.2: /0x[\da-f]*/i | ||||
OCT_NUMBER.2: /0o[0-7]*/i | OCT_NUMBER.2: /0o[0-7]*/i | ||||
BIN_NUMBER.2 : /0b[0-1]*/i | BIN_NUMBER.2 : /0b[0-1]*/i | ||||
FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)(e[-+]?\d+)?|\d+(e[-+]?\d+))/i | |||||
IMAG_NUMBER.2: /\d+j/i | FLOAT_NUMBER "j"i | |||||
FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)([Ee][-+]?\d+)?|\d+([Ee][-+]?\d+))/ | |||||
IMAG_NUMBER.2: /\d+[Jj]/ | FLOAT_NUMBER /[Jj]/ | |||||
// Comma-separated list (with an optional trailing comma) | // Comma-separated list (with an optional trailing comma) | ||||
@@ -1,19 +1,18 @@ | |||||
"Provides Indentation services for languages with indentation similar to Python" | "Provides Indentation services for languages with indentation similar to Python" | ||||
from abc import ABC, abstractmethod | from abc import ABC, abstractmethod | ||||
from typing import List, Iterator | |||||
from .exceptions import LarkError | from .exceptions import LarkError | ||||
from .lark import PostLex | from .lark import PostLex | ||||
from .lexer import Token | from .lexer import Token | ||||
###{standalone | ###{standalone | ||||
from typing import Tuple, List, Iterator, Optional | |||||
class DedentError(LarkError): | class DedentError(LarkError): | ||||
pass | pass | ||||
class Indenter(PostLex, ABC): | class Indenter(PostLex, ABC): | ||||
paren_level: int | paren_level: int | ||||
indent_level: List[int] | indent_level: List[int] | ||||
@@ -75,31 +74,31 @@ class Indenter(PostLex, ABC): | |||||
@property | @property | ||||
@abstractmethod | @abstractmethod | ||||
def NL_type(self) -> str: | def NL_type(self) -> str: | ||||
... | |||||
raise NotImplementedError() | |||||
@property | @property | ||||
@abstractmethod | @abstractmethod | ||||
def OPEN_PAREN_types(self) -> List[str]: | def OPEN_PAREN_types(self) -> List[str]: | ||||
... | |||||
raise NotImplementedError() | |||||
@property | @property | ||||
@abstractmethod | @abstractmethod | ||||
def CLOSE_PAREN_types(self) -> List[str]: | def CLOSE_PAREN_types(self) -> List[str]: | ||||
... | |||||
raise NotImplementedError() | |||||
@property | @property | ||||
@abstractmethod | @abstractmethod | ||||
def INDENT_type(self) -> str: | def INDENT_type(self) -> str: | ||||
... | |||||
raise NotImplementedError() | |||||
@property | @property | ||||
@abstractmethod | @abstractmethod | ||||
def DEDENT_type(self) -> str: | def DEDENT_type(self) -> str: | ||||
... | |||||
raise NotImplementedError() | |||||
@property | @property | ||||
@abstractmethod | @abstractmethod | ||||
def tab_len(self) -> int: | def tab_len(self) -> int: | ||||
... | |||||
raise NotImplementedError() | |||||
###} | ###} |
@@ -1,7 +1,18 @@ | |||||
from abc import ABC, abstractmethod | from abc import ABC, abstractmethod | ||||
import sys, os, pickle, hashlib | import sys, os, pickle, hashlib | ||||
import tempfile | import tempfile | ||||
from typing import ( | |||||
TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, | |||||
Tuple, Iterable, IO, Any, TYPE_CHECKING | |||||
) | |||||
if TYPE_CHECKING: | |||||
from .parsers.lalr_interactive_parser import InteractiveParser | |||||
from .visitors import Transformer | |||||
if sys.version_info >= (3, 8): | |||||
from typing import Literal | |||||
else: | |||||
from typing_extensions import Literal | |||||
from .exceptions import ConfigurationError, assert_config, UnexpectedInput | from .exceptions import ConfigurationError, assert_config, UnexpectedInput | ||||
from .utils import Serialize, SerializeMemoizer, FS, isascii, logger | from .utils import Serialize, SerializeMemoizer, FS, isascii, logger | ||||
from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource | from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource | ||||
@@ -21,18 +32,7 @@ except ImportError: | |||||
###{standalone | ###{standalone | ||||
from typing import ( | |||||
TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, | |||||
Tuple, Iterable, IO, Any, TYPE_CHECKING | |||||
) | |||||
if TYPE_CHECKING: | |||||
from .parsers.lalr_interactive_parser import InteractiveParser | |||||
from .visitors import Transformer | |||||
if sys.version_info >= (3, 8): | |||||
from typing import Literal | |||||
else: | |||||
from typing_extensions import Literal | |||||
class PostLex(ABC): | class PostLex(ABC): | ||||
@abstractmethod | @abstractmethod | ||||
@@ -3,6 +3,13 @@ | |||||
from abc import abstractmethod, ABC | from abc import abstractmethod, ABC | ||||
import re | import re | ||||
from contextlib import suppress | from contextlib import suppress | ||||
from typing import ( | |||||
TypeVar, Type, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, | |||||
Pattern as REPattern, ClassVar, TYPE_CHECKING | |||||
) | |||||
from types import ModuleType | |||||
if TYPE_CHECKING: | |||||
from .common import LexerConf | |||||
from .utils import classify, get_regexp_width, Serialize | from .utils import classify, get_regexp_width, Serialize | ||||
from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken | from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken | ||||
@@ -10,14 +17,6 @@ from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken | |||||
###{standalone | ###{standalone | ||||
from copy import copy | from copy import copy | ||||
from types import ModuleType | |||||
from typing import ( | |||||
TypeVar, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, | |||||
Pattern as REPattern, ClassVar, TYPE_CHECKING | |||||
) | |||||
if TYPE_CHECKING: | |||||
from .common import LexerConf | |||||
class Pattern(Serialize, ABC): | class Pattern(Serialize, ABC): | ||||
@@ -218,7 +217,7 @@ class LineCounter: | |||||
return self.char_pos == other.char_pos and self.newline_char == other.newline_char | return self.char_pos == other.char_pos and self.newline_char == other.newline_char | ||||
def feed(self, token, test_newline=True): | |||||
def feed(self, token: Token, test_newline=True): | |||||
"""Consume a token and calculate the new line & column. | """Consume a token and calculate the new line & column. | ||||
As an optional optimization, set test_newline=False if token doesn't contain a newline. | As an optional optimization, set test_newline=False if token doesn't contain a newline. | ||||
@@ -320,7 +319,7 @@ class Scanner: | |||||
return m.group(0), type_from_index[m.lastindex] | return m.group(0), type_from_index[m.lastindex] | ||||
def _regexp_has_newline(r): | |||||
def _regexp_has_newline(r: str): | |||||
r"""Expressions that may indicate newlines in a regexp: | r"""Expressions that may indicate newlines in a regexp: | ||||
- newlines (\n) | - newlines (\n) | ||||
- escaped newline (\\n) | - escaped newline (\\n) | ||||
@@ -359,7 +358,7 @@ class Lexer(ABC): | |||||
""" | """ | ||||
@abstractmethod | @abstractmethod | ||||
def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]: | def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]: | ||||
... | |||||
return NotImplemented | |||||
def make_lexer_state(self, text): | def make_lexer_state(self, text): | ||||
line_ctr = LineCounter(b'\n' if isinstance(text, bytes) else '\n') | line_ctr = LineCounter(b'\n' if isinstance(text, bytes) else '\n') | ||||
@@ -1,3 +1,5 @@ | |||||
from typing import List | |||||
from .exceptions import GrammarError, ConfigurationError | from .exceptions import GrammarError, ConfigurationError | ||||
from .lexer import Token | from .lexer import Token | ||||
from .tree import Tree | from .tree import Tree | ||||
@@ -151,7 +153,7 @@ def _should_expand(sym): | |||||
return not sym.is_term and sym.name.startswith('_') | return not sym.is_term and sym.name.startswith('_') | ||||
def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices): | |||||
def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices: List[bool]): | |||||
# Prepare empty_indices as: How many Nones to insert at each index? | # Prepare empty_indices as: How many Nones to insert at each index? | ||||
if _empty_indices: | if _empty_indices: | ||||
assert _empty_indices.count(False) == len(expansion) | assert _empty_indices.count(False) == len(expansion) | ||||
@@ -32,20 +32,13 @@ class MakeParsingFrontend: | |||||
self.parser_type = parser_type | self.parser_type = parser_type | ||||
self.lexer_type = lexer_type | self.lexer_type = lexer_type | ||||
def __call__(self, lexer_conf, parser_conf, options): | |||||
assert isinstance(lexer_conf, LexerConf) | |||||
assert isinstance(parser_conf, ParserConf) | |||||
parser_conf.parser_type = self.parser_type | |||||
lexer_conf.lexer_type = self.lexer_type | |||||
return ParsingFrontend(lexer_conf, parser_conf, options) | |||||
def deserialize(self, data, memo, lexer_conf, callbacks, options): | def deserialize(self, data, memo, lexer_conf, callbacks, options): | ||||
parser_conf = ParserConf.deserialize(data['parser_conf'], memo) | parser_conf = ParserConf.deserialize(data['parser_conf'], memo) | ||||
parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug) | parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug) | ||||
parser_conf.callbacks = callbacks | parser_conf.callbacks = callbacks | ||||
return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser) | return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser) | ||||
# ... Continued later in the module | |||||
class ParsingFrontend(Serialize): | class ParsingFrontend(Serialize): | ||||
@@ -237,3 +230,12 @@ class CYK_FrontEnd: | |||||
def _apply_callback(self, tree): | def _apply_callback(self, tree): | ||||
return self.callbacks[tree.rule](tree.children) | return self.callbacks[tree.rule](tree.children) | ||||
class MakeParsingFrontend(MakeParsingFrontend): | |||||
def __call__(self, lexer_conf, parser_conf, options): | |||||
assert isinstance(lexer_conf, LexerConf) | |||||
assert isinstance(parser_conf, ParserConf) | |||||
parser_conf.parser_type = self.parser_type | |||||
lexer_conf.lexer_type = self.lexer_type | |||||
return ParsingFrontend(lexer_conf, parser_conf, options) |
@@ -25,6 +25,13 @@ | |||||
# | # | ||||
from abc import ABC, abstractmethod | from abc import ABC, abstractmethod | ||||
from collections.abc import Sequence | |||||
from types import ModuleType | |||||
from typing import ( | |||||
TypeVar, Generic, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, | |||||
Union, Iterable, IO, TYPE_CHECKING, | |||||
Pattern as REPattern, ClassVar, Set, | |||||
) | |||||
###} | ###} | ||||
import sys | import sys | ||||
@@ -1,3 +1,4 @@ | |||||
try: | try: | ||||
from future_builtins import filter # type: ignore | from future_builtins import filter # type: ignore | ||||
except ImportError: | except ImportError: | ||||
@@ -6,10 +7,7 @@ except ImportError: | |||||
import sys | import sys | ||||
from copy import deepcopy | from copy import deepcopy | ||||
###{standalone | |||||
from collections import OrderedDict | |||||
from typing import List, Callable, Iterator, Union, Optional, Any, TYPE_CHECKING | |||||
from typing import List, Callable, Iterator, Union, Optional, TYPE_CHECKING | |||||
if TYPE_CHECKING: | if TYPE_CHECKING: | ||||
from .lexer import TerminalDef | from .lexer import TerminalDef | ||||
@@ -18,6 +16,9 @@ if TYPE_CHECKING: | |||||
else: | else: | ||||
from typing_extensions import Literal | from typing_extensions import Literal | ||||
###{standalone | |||||
from collections import OrderedDict | |||||
class Meta: | class Meta: | ||||
empty: bool | empty: bool | ||||
@@ -241,7 +241,7 @@ def combine_alternatives(lists): | |||||
try: | try: | ||||
import atomicwrites | import atomicwrites | ||||
except ImportError: | except ImportError: | ||||
atomicwrites = None | |||||
atomicwrites = None # type: ignore | |||||
class FS: | class FS: | ||||
exists = os.path.exists | exists = os.path.exists | ||||
@@ -1,3 +1,4 @@ | |||||
from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional | |||||
from abc import ABC | from abc import ABC | ||||
from functools import wraps | from functools import wraps | ||||
@@ -8,7 +9,6 @@ from .lexer import Token | |||||
###{standalone | ###{standalone | ||||
from inspect import getmembers, getmro | from inspect import getmembers, getmro | ||||
from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional | |||||
_T = TypeVar('_T') | _T = TypeVar('_T') | ||||
_R = TypeVar('_R') | _R = TypeVar('_R') | ||||
@@ -24,7 +24,7 @@ class TestStandalone(TestCase): | |||||
standalone.gen_standalone(Lark(grammar, parser='lalr'), out=code_buf, compress=compress) | standalone.gen_standalone(Lark(grammar, parser='lalr'), out=code_buf, compress=compress) | ||||
code = code_buf.getvalue() | code = code_buf.getvalue() | ||||
context = {'__doc__': None} | |||||
context = {'__doc__': None, '__name__': 'test_standalone'} | |||||
exec(code, context) | exec(code, context) | ||||
return context | return context | ||||