From 111738b8b9502a58159a6fa7e841e95444ff3bd1 Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Fri, 27 Nov 2020 10:46:13 +0200 Subject: [PATCH] Revised handling of ignored tokens in xearley (Issue #768) --- lark/parsers/xearley.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py index d965421..79ac82f 100644 --- a/lark/parsers/xearley.py +++ b/lark/parsers/xearley.py @@ -63,9 +63,10 @@ class Parser(BaseParser): t = Token(item.expect.name, m.group(0), i, text_line, text_column) delayed_matches[i+m.end()].append( (item, i, t) ) - # Remove any items that successfully matched in this pass from the to_scan buffer. - # This ensures we don't carry over tokens that already matched, if we're ignoring below. - to_scan.remove(item) + # XXX The following 3 lines were commented out for causing a bug. See issue #768 + # # Remove any items that successfully matched in this pass from the to_scan buffer. + # # This ensures we don't carry over tokens that already matched, if we're ignoring below. + # to_scan.remove(item) # 3) Process any ignores. This is typically used for e.g. whitespace. # We carry over any unmatched items from the to_scan buffer to be matched again after