Browse Source

Merge pull request #930 from MegaIng/fix-tree_matcher

fix tree_matcher when keep_all_tokens=True by setting sym.filter_out …
tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.12.0
Erez Shinan 3 years ago
committed by GitHub
parent
commit
d637b1179f
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 18 additions and 3 deletions
  1. +4
    -1
      lark/load_grammar.py
  2. +14
    -2
      tests/test_reconstructor.py

+ 4
- 1
lark/load_grammar.py View File

@@ -744,7 +744,10 @@ class Grammar:
else:
exp_options = options

assert all(isinstance(x, Symbol) for x in expansion), expansion
for sym in expansion:
assert isinstance(sym, Symbol)
if sym.is_term and exp_options and exp_options.keep_all_tokens:
sym.filter_out = False
rule = Rule(NonTerminal(name), expansion, i, alias, exp_options)
compiled_rules.append(rule)



+ 14
- 2
tests/test_reconstructor.py View File

@@ -3,6 +3,7 @@
import json
import sys
import unittest
from itertools import product
from unittest import TestCase

from lark import Lark
@@ -20,8 +21,8 @@ def _remove_ws(s):

class TestReconstructor(TestCase):

def assert_reconstruct(self, grammar, code):
parser = Lark(grammar, parser='lalr', maybe_placeholders=False)
def assert_reconstruct(self, grammar, code, **options):
parser = Lark(grammar, parser='lalr', maybe_placeholders=False, **options)
tree = parser.parse(code)
new = Reconstructor(parser).reconstruct(tree)
self.assertEqual(_remove_ws(code), _remove_ws(new))
@@ -142,6 +143,17 @@ class TestReconstructor(TestCase):
new_json = Reconstructor(json_parser).reconstruct(tree)
self.assertEqual(json.loads(new_json), json.loads(test_json))

def test_keep_all_tokens(self):
g = """
start: "a"? _B? c? _d?
_B: "b"
c: "c"
_d: "d"
"""
examples = list(map(''.join, product(('', 'a'), ('', 'b'), ('', 'c'), ('', 'd'), )))
for code in examples:
self.assert_reconstruct(g, code, keep_all_tokens=True)

@unittest.skipIf(sys.version_info < (3, 0), "Python 2 does not play well with Unicode.")
def test_switch_grammar_unicode_terminal(self):
"""


Loading…
Cancel
Save