Merge pull request #994 from lark-parser/v1.0_fixes2

lark/CHANGELOG.md

@@ -4,3 +4,6 @@ v1.0
 
 - `use_accepts` in `UnexpectedInput.match_examples()` is now True by default
 
+- Token priority is now 0 by default
+
+- `v_args(meta=True)` now gives meta as the first argument. i.e. `(meta, children)`

lark/grammar.py

@@ -3,6 +3,8 @@ from typing import Optional, Tuple, ClassVar
 from .utils import Serialize
 
 ###{standalone
+TOKEN_DEFAULT_PRIORITY = 0
+
 
 class Symbol(Serialize):
     __slots__ = ('name',)

lark/lexer.py

@@ -13,6 +13,7 @@ if TYPE_CHECKING:
 
 from .utils import classify, get_regexp_width, Serialize
 from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken
+from .grammar import TOKEN_DEFAULT_PRIORITY
 
 ###{standalone
 from copy import copy
@@ -108,7 +109,7 @@ class TerminalDef(Serialize):
     pattern: Pattern
     priority: int
 
-    def __init__(self, name: str, pattern: Pattern, priority: int=1) -> None:
+    def __init__(self, name: str, pattern: Pattern, priority: int=TOKEN_DEFAULT_PRIORITY) -> None:
         assert isinstance(pattern, Pattern), pattern
         self.name = name
         self.pattern = pattern

lark/load_grammar.py

@@ -15,7 +15,7 @@ from .lexer import Token, TerminalDef, PatternStr, PatternRE
 from .parse_tree_builder import ParseTreeBuilder
 from .parser_frontends import ParsingFrontend
 from .common import LexerConf, ParserConf
-from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
+from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol, TOKEN_DEFAULT_PRIORITY
 from .utils import classify, dedup_list
 from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError, UnexpectedInput
 
@@ -1121,7 +1121,7 @@ class GrammarBuilder:
 
             name = '__IGNORE_%d'% len(self._ignore_names)
             self._ignore_names.append(name)
-            self._definitions[name] = ((), t, 1)
+            self._definitions[name] = ((), t, TOKEN_DEFAULT_PRIORITY)
 
     def _declare(self, *names):
         for name in names:
@@ -1172,7 +1172,7 @@ class GrammarBuilder:
         else:
             name = tree.children[0].value
             params = ()     # TODO terminal templates
-            opts = int(tree.children[1]) if len(tree.children) == 3 else 1 # priority
+            opts = int(tree.children[1]) if len(tree.children) == 3 else TOKEN_DEFAULT_PRIORITY # priority
             exp = tree.children[-1]
 
         if mangle is not None:

lark/parser_frontends.py

@@ -162,7 +162,7 @@ class EarleyRegexpMatcher:
     def __init__(self, lexer_conf):
         self.regexps = {}
         for t in lexer_conf.terminals:
-            if t.priority != 1:
+            if t.priority:
                 raise GrammarError("Dynamic Earley doesn't support weights on terminals", t, t.priority)
             regexp = t.pattern.to_regexp()
             try:

tests/test_nearley/nearley

@@ -1 +1 @@
-Subproject commit a46b37471db486db0f6e1ce6a2934fb238346b44
+Subproject commit 326831689826cb1b9a4d21d1ce0d5db9278e9636