Merge pull request #963 from lark-parser/v1.0-merge-master2

.github/workflows/tests.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9.0-rc - 3.9, pypy3]
+        python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-rc - 3.10, pypy3]
 
     steps:
       - uses: actions/checkout@v2

.travis.yml

@@ -1,13 +0,0 @@
-dist: xenial
-language: python
-python:
-  - "3.4"
-  - "3.5"
-  - "3.6"
-  - "3.7"
-  - "3.8"
-  - "3.9-dev"
-  - "pypy3.5-6.0"
-install: pip install tox-travis
-script:
-  - tox

README.md

@@ -37,7 +37,7 @@ Most importantly, Lark will save you time and prevent you from getting parsing h
 
 Lark has no dependencies.
 
-[![Build Status](https://travis-ci.org/lark-parser/lark.svg?branch=master)](https://travis-ci.org/lark-parser/lark)
+[![Tests](https://github.com/lark-parser/lark/actions/workflows/tests.yml/badge.svg)](https://github.com/lark-parser/lark/actions/workflows/tests.yml)
 
 ### Syntax Highlighting
 

lark/lexer.py

@@ -282,7 +282,6 @@ def _create_unless(terminals, g_regex_flags, re_, use_bytes):
     return new_terminals, callback
 
 
-
 class Scanner:
     def __init__(self, terminals, g_regex_flags, re_, use_bytes, match_whole=False):
         self.terminals = terminals

lark/load_grammar.py

@@ -739,7 +739,10 @@ class Grammar:
                 else:
                     exp_options = options
 
-                assert all(isinstance(x, Symbol) for x in expansion), expansion
+                for sym in expansion:
+                    assert isinstance(sym, Symbol)
+                    if sym.is_term and exp_options and exp_options.keep_all_tokens:
+                        sym.filter_out = False
                 rule = Rule(NonTerminal(name), expansion, i, alias, exp_options)
                 compiled_rules.append(rule)
 

setup.cfg

@@ -5,6 +5,4 @@ zip_safe=
 universal = 1
 
 [metadata]
-description-file = README.md
 license_file = LICENSE
-

tests/test_reconstructor.py

@@ -3,6 +3,7 @@
 import json
 import sys
 import unittest
+from itertools import product
 from unittest import TestCase
 
 from lark import Lark
@@ -20,8 +21,8 @@ def _remove_ws(s):
 
 class TestReconstructor(TestCase):
 
-    def assert_reconstruct(self, grammar, code):
-        parser = Lark(grammar, parser='lalr', maybe_placeholders=False)
+    def assert_reconstruct(self, grammar, code, **options):
+        parser = Lark(grammar, parser='lalr', maybe_placeholders=False, **options)
         tree = parser.parse(code)
         new = Reconstructor(parser).reconstruct(tree)
         self.assertEqual(_remove_ws(code), _remove_ws(new))
@@ -142,6 +143,17 @@ class TestReconstructor(TestCase):
         new_json = Reconstructor(json_parser).reconstruct(tree)
         self.assertEqual(json.loads(new_json), json.loads(test_json))
 
+    def test_keep_all_tokens(self):
+        g = """
+        start: "a"? _B? c? _d?
+        _B: "b"
+        c: "c"
+        _d: "d"
+        """
+        examples = list(map(''.join, product(('', 'a'), ('', 'b'), ('', 'c'), ('', 'd'), )))
+        for code in examples:
+            self.assert_reconstruct(g, code, keep_all_tokens=True)
+
     @unittest.skipIf(sys.version_info < (3, 0), "Python 2 does not play well with Unicode.")
     def test_switch_grammar_unicode_terminal(self):
         """

tox.ini

@@ -2,13 +2,6 @@
 envlist = py36, py37, py38, py39, pypy, pypy3
 skip_missing_interpreters=true
 
-[travis]
-3.6 = py36
-3.7 = py37
-3.8 = py38
-3.9 = py39
-pypy3 = pypy3
-
 [testenv]
 whitelist_externals = git
 deps =