Merge pull request #964 from lark-parser/1.0b

More updates to v1.0

lark/common.py

@@ -1,14 +1,14 @@
-from types import ModuleType
 from copy import deepcopy
+from types import ModuleType
+from typing import Callable, Collection, Dict, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .lark import PostLex
 
 from .utils import Serialize
 from .lexer import TerminalDef, Token
 
 ###{standalone
-from typing import Any, Callable, Collection, Dict, Optional, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from .lark import PostLex
 
 _Callback = Callable[[Token], Token]
 

lark/exceptions.py

@@ -1,15 +1,13 @@
 from .utils import logger, NO_VALUE
-
-
-###{standalone
-
-from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, TYPE_CHECKING
+from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, Collection, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .lexer import Token
     from .parsers.lalr_interactive_parser import InteractiveParser
     from .tree import Tree
 
+###{standalone
+
 class LarkError(Exception):
     pass
 
@@ -18,7 +16,7 @@ class ConfigurationError(LarkError, ValueError):
     pass
 
 
-def assert_config(value, options, msg='Got %r, expected one of %s'):
+def assert_config(value, options: Collection, msg='Got %r, expected one of %s'):
     if value not in options:
         raise ConfigurationError(msg % (value, options))
 

lark/grammar.py

@@ -1,10 +1,9 @@
+from typing import Optional, Tuple, ClassVar
 
 from .utils import Serialize
 
 ###{standalone
 
-from typing import Optional, Tuple, ClassVar
-
 class Symbol(Serialize):
     __slots__ = ('name',)
 

lark/grammars/python.lark

@@ -10,8 +10,8 @@ DEC_NUMBER: /0|[1-9][\d_]*/i
 HEX_NUMBER.2: /0x[\da-f]*/i
 OCT_NUMBER.2: /0o[0-7]*/i
 BIN_NUMBER.2 : /0b[0-1]*/i
-FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)(e[-+]?\d+)?|\d+(e[-+]?\d+))/i
-IMAG_NUMBER.2: /\d+j/i | FLOAT_NUMBER "j"i
+FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)([Ee][-+]?\d+)?|\d+([Ee][-+]?\d+))/
+IMAG_NUMBER.2: /\d+[Jj]/ | FLOAT_NUMBER /[Jj]/
 
 
 // Comma-separated list (with an optional trailing comma)

lark/indenter.py

@@ -1,19 +1,18 @@
 "Provides Indentation services for languages with indentation similar to Python"
 
 from abc import ABC, abstractmethod
+from typing import List, Iterator
 
 from .exceptions import LarkError
 from .lark import PostLex
 from .lexer import Token
 
 ###{standalone
-from typing import Tuple, List, Iterator, Optional
 
 class DedentError(LarkError):
     pass
 
 class Indenter(PostLex, ABC):
-
     paren_level: int
     indent_level: List[int]
 
@@ -75,31 +74,31 @@ class Indenter(PostLex, ABC):
     @property
     @abstractmethod
     def NL_type(self) -> str:
-        ...
+        raise NotImplementedError()
 
     @property
     @abstractmethod
     def OPEN_PAREN_types(self) -> List[str]:
-        ...
+        raise NotImplementedError()
 
     @property
     @abstractmethod
     def CLOSE_PAREN_types(self) -> List[str]:
-        ...
+        raise NotImplementedError()
 
     @property
     @abstractmethod
     def INDENT_type(self) -> str:
-        ...
+        raise NotImplementedError()
 
     @property
     @abstractmethod
     def DEDENT_type(self) -> str:
-        ...
+        raise NotImplementedError()
 
     @property
     @abstractmethod
     def tab_len(self) -> int:
-        ...
+        raise NotImplementedError()
 
 ###}

lark/lark.py

@@ -1,7 +1,18 @@
 from abc import ABC, abstractmethod
 import sys, os, pickle, hashlib
 import tempfile
-
+from typing import (
+    TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional,
+    Tuple, Iterable, IO, Any, TYPE_CHECKING
+)
+if TYPE_CHECKING:
+    from .parsers.lalr_interactive_parser import InteractiveParser
+    from .visitors import Transformer
+    if sys.version_info >= (3, 8):
+        from typing import Literal
+    else:
+        from typing_extensions import Literal
+        
 from .exceptions import ConfigurationError, assert_config, UnexpectedInput
 from .utils import Serialize, SerializeMemoizer, FS, isascii, logger
 from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource
@@ -21,18 +32,7 @@ except ImportError:
 
 
 ###{standalone
-from typing import (
-    TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional,
-    Tuple, Iterable, IO, Any, TYPE_CHECKING
-)
 
-if TYPE_CHECKING:
-    from .parsers.lalr_interactive_parser import InteractiveParser
-    from .visitors import Transformer
-    if sys.version_info >= (3, 8):
-        from typing import Literal
-    else:
-        from typing_extensions import Literal
 
 class PostLex(ABC):
     @abstractmethod

lark/lexer.py

@@ -3,6 +3,13 @@
 from abc import abstractmethod, ABC
 import re
 from contextlib import suppress
+from typing import (
+    TypeVar, Type, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
+    Pattern as REPattern, ClassVar, TYPE_CHECKING
+)
+from types import ModuleType
+if TYPE_CHECKING:
+    from .common import LexerConf
 
 from .utils import classify, get_regexp_width, Serialize
 from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken
@@ -10,14 +17,6 @@ from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken
 ###{standalone
 from copy import copy
 
-from types import ModuleType
-from typing import (
-    TypeVar, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
-    Pattern as REPattern, ClassVar, TYPE_CHECKING
-)
-
-if TYPE_CHECKING:
-    from .common import LexerConf
 
 class Pattern(Serialize, ABC):
 
@@ -218,7 +217,7 @@ class LineCounter:
 
         return self.char_pos == other.char_pos and self.newline_char == other.newline_char
 
-    def feed(self, token, test_newline=True):
+    def feed(self, token: Token, test_newline=True):
         """Consume a token and calculate the new line & column.
 
         As an optional optimization, set test_newline=False if token doesn't contain a newline.
@@ -320,7 +319,7 @@ class Scanner:
                 return m.group(0), type_from_index[m.lastindex]
 
 
-def _regexp_has_newline(r):
+def _regexp_has_newline(r: str):
     r"""Expressions that may indicate newlines in a regexp:
         - newlines (\n)
         - escaped newline (\\n)
@@ -359,7 +358,7 @@ class Lexer(ABC):
     """
     @abstractmethod
     def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]:
-        ...
+        return NotImplemented
 
     def make_lexer_state(self, text):
         line_ctr = LineCounter(b'\n' if isinstance(text, bytes) else '\n')

lark/parse_tree_builder.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from .exceptions import GrammarError, ConfigurationError
 from .lexer import Token
 from .tree import Tree
@@ -151,7 +153,7 @@ def _should_expand(sym):
     return not sym.is_term and sym.name.startswith('_')
 
 
-def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices):
+def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices: List[bool]):
     # Prepare empty_indices as: How many Nones to insert at each index?
     if _empty_indices:
         assert _empty_indices.count(False) == len(expansion)

lark/parser_frontends.py

@@ -32,20 +32,13 @@ class MakeParsingFrontend:
         self.parser_type = parser_type
         self.lexer_type = lexer_type
 
-    def __call__(self, lexer_conf, parser_conf, options):
-        assert isinstance(lexer_conf, LexerConf)
-        assert isinstance(parser_conf, ParserConf)
-        parser_conf.parser_type = self.parser_type
-        lexer_conf.lexer_type = self.lexer_type
-        return ParsingFrontend(lexer_conf, parser_conf, options)
-
     def deserialize(self, data, memo, lexer_conf, callbacks, options):
         parser_conf = ParserConf.deserialize(data['parser_conf'], memo)
         parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug)
         parser_conf.callbacks = callbacks
         return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser)
 
-
+    # ... Continued later in the module
 
 
 class ParsingFrontend(Serialize):
@@ -237,3 +230,12 @@ class CYK_FrontEnd:
 
     def _apply_callback(self, tree):
         return self.callbacks[tree.rule](tree.children)
+
+
+class MakeParsingFrontend(MakeParsingFrontend):
+    def __call__(self, lexer_conf, parser_conf, options):
+        assert isinstance(lexer_conf, LexerConf)
+        assert isinstance(parser_conf, ParserConf)
+        parser_conf.parser_type = self.parser_type
+        lexer_conf.lexer_type = self.lexer_type
+        return ParsingFrontend(lexer_conf, parser_conf, options)

lark/tools/standalone.py

@@ -25,6 +25,13 @@
 #
 
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
+from types import ModuleType
+from typing import (
+    TypeVar, Generic, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
+    Union, Iterable, IO, TYPE_CHECKING,
+    Pattern as REPattern, ClassVar, Set,
+)
 ###}
 
 import sys

lark/tree.py

@@ -1,3 +1,4 @@
+
 try:
     from future_builtins import filter  # type: ignore
 except ImportError:
@@ -6,10 +7,7 @@ except ImportError:
 import sys
 from copy import deepcopy
 
-
-###{standalone
-from collections import OrderedDict
-from typing import List, Callable, Iterator, Union, Optional, Any, TYPE_CHECKING
+from typing import List, Callable, Iterator, Union, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .lexer import TerminalDef
@@ -18,6 +16,9 @@ if TYPE_CHECKING:
     else:
         from typing_extensions import Literal
 
+###{standalone
+from collections import OrderedDict
+
 class Meta:
 
     empty: bool

lark/utils.py

@@ -241,7 +241,7 @@ def combine_alternatives(lists):
 try:
     import atomicwrites
 except ImportError:
-    atomicwrites = None
+    atomicwrites = None  # type: ignore
 
 class FS:
     exists = os.path.exists

lark/visitors.py

@@ -1,3 +1,4 @@
+from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional
 from abc import ABC
 from functools import wraps
 
@@ -8,7 +9,6 @@ from .lexer import Token
 
 ###{standalone
 from inspect import getmembers, getmro
-from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional
 
 _T = TypeVar('_T')
 _R = TypeVar('_R')

tests/test_tools.py

@@ -24,7 +24,7 @@ class TestStandalone(TestCase):
         standalone.gen_standalone(Lark(grammar, parser='lalr'), out=code_buf, compress=compress)
         code = code_buf.getvalue()
 
-        context = {'__doc__': None}
+        context = {'__doc__': None, '__name__': 'test_standalone'}
         exec(code, context)
         return context