From 820e185dca576f6255fd37d00032a0d7fbf0edd3 Mon Sep 17 00:00:00 2001 From: Chanic Panic Date: Thu, 12 Nov 2020 19:47:53 -0800 Subject: [PATCH 1/2] Fix IndexError (issue #754) --- lark/parsers/earley_forest.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py index 1a71379..03c4573 100644 --- a/lark/parsers/earley_forest.py +++ b/lark/parsers/earley_forest.py @@ -459,15 +459,20 @@ class PackedData(): that comes from the left child and the right child. """ + class _NoData(): + pass + + NO_DATA = _NoData() + def __init__(self, node, data): - self.left = None - self.right = None + self.left = self.NO_DATA + self.right = self.NO_DATA if data: - if node.left: + if node.left is not None: self.left = data[0] - if len(data) > 1 and node.right: + if len(data) > 1: self.right = data[1] - elif node.right: + else: self.right = data[0] class ForestToParseTree(ForestTransformer): @@ -558,12 +563,12 @@ class ForestToParseTree(ForestTransformer): children = [] assert len(data) <= 2 data = PackedData(node, data) - if data.left is not None: + if data.left is not PackedData.NO_DATA: if node.left.is_intermediate and isinstance(data.left, list): children += data.left else: children.append(data.left) - if data.right is not None: + if data.right is not PackedData.NO_DATA: children.append(data.right) if node.parent.is_intermediate: return children From 13abf7ca3a89c548fd782e08236d65124f4c7ccb Mon Sep 17 00:00:00 2001 From: Chanic Panic Date: Fri, 13 Nov 2020 12:10:19 -0800 Subject: [PATCH 2/2] Add test for custom indenter --- tests/test_parser.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_parser.py b/tests/test_parser.py index 6d0981f..6fe2946 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -37,6 +37,7 @@ from lark.tree import Tree from lark.visitors import Transformer, Transformer_InPlace, v_args from lark.grammar import Rule from lark.lexer import TerminalDef, Lexer, TraditionalLexer +from lark.indenter import Indenter logger.setLevel(logging.INFO) @@ -1876,6 +1877,28 @@ def _make_parser_test(LEXER, PARSER): tree = parser.parse(test_file) self.assertEqual(tree.children, [Token('B', 'A')]) + @unittest.skipIf(LEXER=='dynamic', "%declare/postlex doesn't work with dynamic") + def test_postlex_indenter(self): + class CustomIndenter(Indenter): + NL_type = 'NEWLINE' + OPEN_PAREN_types = [] + CLOSE_PAREN_types = [] + INDENT_type = 'INDENT' + DEDENT_type = 'DEDENT' + tab_len = 8 + + grammar = r""" + start: "a" NEWLINE INDENT "b" NEWLINE DEDENT + + NEWLINE: ( /\r?\n */ )+ + + %ignore " "+ + %declare INDENT DEDENT + """ + + parser = _Lark(grammar, postlex=CustomIndenter()) + parser.parse("a\n b\n") + def test_import_custom_sources(self): custom_loader = FromPackageLoader('tests', ('grammars', ))