Browse Source

Merge pull request #994 from lark-parser/v1.0_fixes2

remotes/origin/gm/2021-09-23T00Z/github.com--lark-parser-lark/master
Erez Shinan 3 years ago
committed by GitHub
parent
commit
77b07bbf23
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 12 additions and 6 deletions
  1. +3
    -0
      lark/CHANGELOG.md
  2. +2
    -0
      lark/grammar.py
  3. +2
    -1
      lark/lexer.py
  4. +3
    -3
      lark/load_grammar.py
  5. +1
    -1
      lark/parser_frontends.py
  6. +1
    -1
      tests/test_nearley/nearley

+ 3
- 0
lark/CHANGELOG.md View File

@@ -4,3 +4,6 @@ v1.0
- `use_accepts` in `UnexpectedInput.match_examples()` is now True by default
- Token priority is now 0 by default
- `v_args(meta=True)` now gives meta as the first argument. i.e. `(meta, children)`

+ 2
- 0
lark/grammar.py View File

@@ -3,6 +3,8 @@ from typing import Optional, Tuple, ClassVar
from .utils import Serialize

###{standalone
TOKEN_DEFAULT_PRIORITY = 0


class Symbol(Serialize):
__slots__ = ('name',)


+ 2
- 1
lark/lexer.py View File

@@ -13,6 +13,7 @@ if TYPE_CHECKING:

from .utils import classify, get_regexp_width, Serialize
from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken
from .grammar import TOKEN_DEFAULT_PRIORITY

###{standalone
from copy import copy
@@ -108,7 +109,7 @@ class TerminalDef(Serialize):
pattern: Pattern
priority: int

def __init__(self, name: str, pattern: Pattern, priority: int=1) -> None:
def __init__(self, name: str, pattern: Pattern, priority: int=TOKEN_DEFAULT_PRIORITY) -> None:
assert isinstance(pattern, Pattern), pattern
self.name = name
self.pattern = pattern


+ 3
- 3
lark/load_grammar.py View File

@@ -15,7 +15,7 @@ from .lexer import Token, TerminalDef, PatternStr, PatternRE
from .parse_tree_builder import ParseTreeBuilder
from .parser_frontends import ParsingFrontend
from .common import LexerConf, ParserConf
from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol, TOKEN_DEFAULT_PRIORITY
from .utils import classify, dedup_list
from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError, UnexpectedInput

@@ -1121,7 +1121,7 @@ class GrammarBuilder:

name = '__IGNORE_%d'% len(self._ignore_names)
self._ignore_names.append(name)
self._definitions[name] = ((), t, 1)
self._definitions[name] = ((), t, TOKEN_DEFAULT_PRIORITY)

def _declare(self, *names):
for name in names:
@@ -1172,7 +1172,7 @@ class GrammarBuilder:
else:
name = tree.children[0].value
params = () # TODO terminal templates
opts = int(tree.children[1]) if len(tree.children) == 3 else 1 # priority
opts = int(tree.children[1]) if len(tree.children) == 3 else TOKEN_DEFAULT_PRIORITY # priority
exp = tree.children[-1]

if mangle is not None:


+ 1
- 1
lark/parser_frontends.py View File

@@ -162,7 +162,7 @@ class EarleyRegexpMatcher:
def __init__(self, lexer_conf):
self.regexps = {}
for t in lexer_conf.terminals:
if t.priority != 1:
if t.priority:
raise GrammarError("Dynamic Earley doesn't support weights on terminals", t, t.priority)
regexp = t.pattern.to_regexp()
try:


+ 1
- 1
tests/test_nearley/nearley

@@ -1 +1 @@
Subproject commit a46b37471db486db0f6e1ce6a2934fb238346b44
Subproject commit 326831689826cb1b9a4d21d1ce0d5db9278e9636

Loading…
Cancel
Save