diff --git a/lark/CHANGELOG.md b/lark/CHANGELOG.md index 9fdb072..1c547e4 100644 --- a/lark/CHANGELOG.md +++ b/lark/CHANGELOG.md @@ -4,3 +4,6 @@ v1.0 - `use_accepts` in `UnexpectedInput.match_examples()` is now True by default +- Token priority is now 0 by default + +- `v_args(meta=True)` now gives meta as the first argument. i.e. `(meta, children)` \ No newline at end of file diff --git a/lark/grammar.py b/lark/grammar.py index 3d6f0ff..108f347 100644 --- a/lark/grammar.py +++ b/lark/grammar.py @@ -3,6 +3,8 @@ from typing import Optional, Tuple, ClassVar from .utils import Serialize ###{standalone +TOKEN_DEFAULT_PRIORITY = 0 + class Symbol(Serialize): __slots__ = ('name',) diff --git a/lark/lexer.py b/lark/lexer.py index 0b7bef8..292bb35 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from .utils import classify, get_regexp_width, Serialize from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken +from .grammar import TOKEN_DEFAULT_PRIORITY ###{standalone from copy import copy @@ -108,7 +109,7 @@ class TerminalDef(Serialize): pattern: Pattern priority: int - def __init__(self, name: str, pattern: Pattern, priority: int=1) -> None: + def __init__(self, name: str, pattern: Pattern, priority: int=TOKEN_DEFAULT_PRIORITY) -> None: assert isinstance(pattern, Pattern), pattern self.name = name self.pattern = pattern diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 4d73e61..461b21e 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -15,7 +15,7 @@ from .lexer import Token, TerminalDef, PatternStr, PatternRE from .parse_tree_builder import ParseTreeBuilder from .parser_frontends import ParsingFrontend from .common import LexerConf, ParserConf -from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol +from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol, TOKEN_DEFAULT_PRIORITY from .utils import classify, dedup_list from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError, UnexpectedInput @@ -1121,7 +1121,7 @@ class GrammarBuilder: name = '__IGNORE_%d'% len(self._ignore_names) self._ignore_names.append(name) - self._definitions[name] = ((), t, 1) + self._definitions[name] = ((), t, TOKEN_DEFAULT_PRIORITY) def _declare(self, *names): for name in names: @@ -1172,7 +1172,7 @@ class GrammarBuilder: else: name = tree.children[0].value params = () # TODO terminal templates - opts = int(tree.children[1]) if len(tree.children) == 3 else 1 # priority + opts = int(tree.children[1]) if len(tree.children) == 3 else TOKEN_DEFAULT_PRIORITY # priority exp = tree.children[-1] if mangle is not None: diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index 37d59ae..f79ea36 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -162,7 +162,7 @@ class EarleyRegexpMatcher: def __init__(self, lexer_conf): self.regexps = {} for t in lexer_conf.terminals: - if t.priority != 1: + if t.priority: raise GrammarError("Dynamic Earley doesn't support weights on terminals", t, t.priority) regexp = t.pattern.to_regexp() try: diff --git a/tests/test_nearley/nearley b/tests/test_nearley/nearley index a46b374..3268316 160000 --- a/tests/test_nearley/nearley +++ b/tests/test_nearley/nearley @@ -1 +1 @@ -Subproject commit a46b37471db486db0f6e1ce6a2934fb238346b44 +Subproject commit 326831689826cb1b9a4d21d1ce0d5db9278e9636