@@ -6,7 +6,7 @@ jobs: | |||||
runs-on: ubuntu-latest | runs-on: ubuntu-latest | ||||
strategy: | strategy: | ||||
matrix: | matrix: | ||||
python-version: [3.6, 3.7, 3.8, 3.9.0-rc - 3.9, pypy3] | |||||
python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-rc - 3.10, pypy3] | |||||
steps: | steps: | ||||
- uses: actions/checkout@v2 | - uses: actions/checkout@v2 | ||||
@@ -1,11 +0,0 @@ | |||||
dist: xenial | |||||
language: python | |||||
python: | |||||
- "3.6" | |||||
- "3.7" | |||||
- "3.8" | |||||
- "3.9-dev" | |||||
- "pypy3.5-6.0" | |||||
install: pip install tox-travis | |||||
script: | |||||
- tox |
@@ -37,7 +37,7 @@ Most importantly, Lark will save you time and prevent you from getting parsing h | |||||
Lark has no dependencies. | Lark has no dependencies. | ||||
[![Build Status](https://travis-ci.org/lark-parser/lark.svg?branch=master)](https://travis-ci.org/lark-parser/lark) | |||||
[![Tests](https://github.com/lark-parser/lark/actions/workflows/tests.yml/badge.svg)](https://github.com/lark-parser/lark/actions/workflows/tests.yml) | |||||
### Syntax Highlighting | ### Syntax Highlighting | ||||
@@ -739,7 +739,10 @@ class Grammar: | |||||
else: | else: | ||||
exp_options = options | exp_options = options | ||||
assert all(isinstance(x, Symbol) for x in expansion), expansion | |||||
for sym in expansion: | |||||
assert isinstance(sym, Symbol) | |||||
if sym.is_term and exp_options and exp_options.keep_all_tokens: | |||||
sym.filter_out = False | |||||
rule = Rule(NonTerminal(name), expansion, i, alias, exp_options) | rule = Rule(NonTerminal(name), expansion, i, alias, exp_options) | ||||
compiled_rules.append(rule) | compiled_rules.append(rule) | ||||
@@ -5,6 +5,4 @@ zip_safe= | |||||
universal = 1 | universal = 1 | ||||
[metadata] | [metadata] | ||||
description-file = README.md | |||||
license_file = LICENSE | license_file = LICENSE | ||||
@@ -3,6 +3,7 @@ | |||||
import json | import json | ||||
import sys | import sys | ||||
import unittest | import unittest | ||||
from itertools import product | |||||
from unittest import TestCase | from unittest import TestCase | ||||
from lark import Lark | from lark import Lark | ||||
@@ -20,8 +21,8 @@ def _remove_ws(s): | |||||
class TestReconstructor(TestCase): | class TestReconstructor(TestCase): | ||||
def assert_reconstruct(self, grammar, code): | |||||
parser = Lark(grammar, parser='lalr', maybe_placeholders=False) | |||||
def assert_reconstruct(self, grammar, code, **options): | |||||
parser = Lark(grammar, parser='lalr', maybe_placeholders=False, **options) | |||||
tree = parser.parse(code) | tree = parser.parse(code) | ||||
new = Reconstructor(parser).reconstruct(tree) | new = Reconstructor(parser).reconstruct(tree) | ||||
self.assertEqual(_remove_ws(code), _remove_ws(new)) | self.assertEqual(_remove_ws(code), _remove_ws(new)) | ||||
@@ -142,6 +143,17 @@ class TestReconstructor(TestCase): | |||||
new_json = Reconstructor(json_parser).reconstruct(tree) | new_json = Reconstructor(json_parser).reconstruct(tree) | ||||
self.assertEqual(json.loads(new_json), json.loads(test_json)) | self.assertEqual(json.loads(new_json), json.loads(test_json)) | ||||
def test_keep_all_tokens(self): | |||||
g = """ | |||||
start: "a"? _B? c? _d? | |||||
_B: "b" | |||||
c: "c" | |||||
_d: "d" | |||||
""" | |||||
examples = list(map(''.join, product(('', 'a'), ('', 'b'), ('', 'c'), ('', 'd'), ))) | |||||
for code in examples: | |||||
self.assert_reconstruct(g, code, keep_all_tokens=True) | |||||
@unittest.skipIf(sys.version_info < (3, 0), "Python 2 does not play well with Unicode.") | @unittest.skipIf(sys.version_info < (3, 0), "Python 2 does not play well with Unicode.") | ||||
def test_switch_grammar_unicode_terminal(self): | def test_switch_grammar_unicode_terminal(self): | ||||
""" | """ | ||||
@@ -2,13 +2,6 @@ | |||||
envlist = py36, py37, py38, py39, pypy, pypy3 | envlist = py36, py37, py38, py39, pypy, pypy3 | ||||
skip_missing_interpreters=true | skip_missing_interpreters=true | ||||
[travis] | |||||
3.6 = py36 | |||||
3.7 = py37 | |||||
3.8 = py38 | |||||
3.9 = py39 | |||||
pypy3 = pypy3 | |||||
[testenv] | [testenv] | ||||
whitelist_externals = git | whitelist_externals = git | ||||
deps = | deps = | ||||