diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a635e58..a98a00b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9.0-rc - 3.9, pypy3] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-rc - 3.10, pypy3] steps: - uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5784ae8..0000000 --- a/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -dist: xenial -language: python -python: - - "3.6" - - "3.7" - - "3.8" - - "3.9-dev" - - "pypy3.5-6.0" -install: pip install tox-travis -script: - - tox diff --git a/README.md b/README.md index 82f6148..70be4fe 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Most importantly, Lark will save you time and prevent you from getting parsing h Lark has no dependencies. -[![Build Status](https://travis-ci.org/lark-parser/lark.svg?branch=master)](https://travis-ci.org/lark-parser/lark) +[![Tests](https://github.com/lark-parser/lark/actions/workflows/tests.yml/badge.svg)](https://github.com/lark-parser/lark/actions/workflows/tests.yml) ### Syntax Highlighting diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 3b46426..c0503e6 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -739,7 +739,10 @@ class Grammar: else: exp_options = options - assert all(isinstance(x, Symbol) for x in expansion), expansion + for sym in expansion: + assert isinstance(sym, Symbol) + if sym.is_term and exp_options and exp_options.keep_all_tokens: + sym.filter_out = False rule = Rule(NonTerminal(name), expansion, i, alias, exp_options) compiled_rules.append(rule) diff --git a/setup.cfg b/setup.cfg index 6ddead9..6d71f28 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,4 @@ zip_safe= universal = 1 [metadata] -description-file = README.md license_file = LICENSE - diff --git a/tests/test_reconstructor.py b/tests/test_reconstructor.py index f132312..e2f2dbe 100644 --- a/tests/test_reconstructor.py +++ b/tests/test_reconstructor.py @@ -3,6 +3,7 @@ import json import sys import unittest +from itertools import product from unittest import TestCase from lark import Lark @@ -20,8 +21,8 @@ def _remove_ws(s): class TestReconstructor(TestCase): - def assert_reconstruct(self, grammar, code): - parser = Lark(grammar, parser='lalr', maybe_placeholders=False) + def assert_reconstruct(self, grammar, code, **options): + parser = Lark(grammar, parser='lalr', maybe_placeholders=False, **options) tree = parser.parse(code) new = Reconstructor(parser).reconstruct(tree) self.assertEqual(_remove_ws(code), _remove_ws(new)) @@ -142,6 +143,17 @@ class TestReconstructor(TestCase): new_json = Reconstructor(json_parser).reconstruct(tree) self.assertEqual(json.loads(new_json), json.loads(test_json)) + def test_keep_all_tokens(self): + g = """ + start: "a"? _B? c? _d? + _B: "b" + c: "c" + _d: "d" + """ + examples = list(map(''.join, product(('', 'a'), ('', 'b'), ('', 'c'), ('', 'd'), ))) + for code in examples: + self.assert_reconstruct(g, code, keep_all_tokens=True) + @unittest.skipIf(sys.version_info < (3, 0), "Python 2 does not play well with Unicode.") def test_switch_grammar_unicode_terminal(self): """ diff --git a/tox.ini b/tox.ini index 33f9f64..04840eb 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,6 @@ envlist = py36, py37, py38, py39, pypy, pypy3 skip_missing_interpreters=true -[travis] -3.6 = py36 -3.7 = py37 -3.8 = py38 -3.9 = py39 -pypy3 = pypy3 - [testenv] whitelist_externals = git deps =