From a13cfcef55f6460b9b8897e9c313b9bcb4c80b33 Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Mon, 5 Jul 2021 12:08:38 +0300 Subject: [PATCH 1/4] Bugfix in propagate_positions: Corrected to account for 'container nodes' --- lark/parse_tree_builder.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index b4929c6..39d3510 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -30,23 +30,36 @@ class PropagatePositions: def __call__(self, children): res = self.node_builder(children) - # local reference to Tree.meta reduces number of presence checks if isinstance(res, Tree): + # Calculate positions while the tree is streaming, according to the rule: + # - nodes start at the start of their first child's container, + # and end at the end of their last child's container. + # Containers are nodes that take up space in text, but have been inlined in the tree. + res_meta = res.meta first_meta = self._pp_get_meta(children) if first_meta is not None: - res_meta.line = first_meta.line - res_meta.column = first_meta.column - res_meta.start_pos = first_meta.start_pos - res_meta.empty = False + # meta was already set, probably because the rule has been inlined (e.g. `?rule`) + if not hasattr(res_meta, 'line'): + res_meta.line = getattr(first_meta, 'container_line', first_meta.line) + res_meta.column = getattr(first_meta, 'container_column', first_meta.column) + res_meta.start_pos = getattr(first_meta, 'container_start_pos', first_meta.start_pos) + res_meta.empty = False + + res_meta.container_line = getattr(first_meta, 'container_line', first_meta.line) + res_meta.container_column = getattr(first_meta, 'container_column', first_meta.column) last_meta = self._pp_get_meta(reversed(children)) if last_meta is not None: - res_meta.end_line = last_meta.end_line - res_meta.end_column = last_meta.end_column - res_meta.end_pos = last_meta.end_pos - res_meta.empty = False + if not hasattr(res_meta, 'end_line'): + res_meta.end_line = getattr(last_meta, 'container_end_line', last_meta.end_line) + res_meta.end_column = getattr(last_meta, 'container_end_column', last_meta.end_column) + res_meta.end_pos = getattr(last_meta, 'container_end_pos', last_meta.end_pos) + res_meta.empty = False + + res_meta.container_end_line = getattr(last_meta, 'container_end_line', last_meta.end_line) + res_meta.container_end_column = getattr(last_meta, 'container_end_column', last_meta.end_column) return res From d7d02e930899048a18b094d798080e59c5b9af9b Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Mon, 5 Jul 2021 12:11:03 +0300 Subject: [PATCH 2/4] Tiny comment fix --- lark/parse_tree_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 39d3510..286038e 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -40,8 +40,8 @@ class PropagatePositions: first_meta = self._pp_get_meta(children) if first_meta is not None: - # meta was already set, probably because the rule has been inlined (e.g. `?rule`) if not hasattr(res_meta, 'line'): + # meta was already set, probably because the rule has been inlined (e.g. `?rule`) res_meta.line = getattr(first_meta, 'container_line', first_meta.line) res_meta.column = getattr(first_meta, 'container_column', first_meta.column) res_meta.start_pos = getattr(first_meta, 'container_start_pos', first_meta.start_pos) From c953dd9505dbba1bd8fbded0077a040a1ce0e5b5 Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Mon, 5 Jul 2021 13:15:29 +0300 Subject: [PATCH 3/4] Tests: Added a test case demonstrating the need for calculating containers --- tests/test_parser.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_parser.py b/tests/test_parser.py index ff4e064..40ed131 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -94,6 +94,26 @@ class TestParsers(unittest.TestCase): r = g.parse('a') self.assertEqual( r.children[0].meta.line, 1 ) + def test_propagate_positions2(self): + g = Lark("""start: a + a: b + ?b: "(" t ")" + !t: "t" + """, propagate_positions=True) + + start = g.parse("(t)") + a ,= start.children + t ,= a.children + assert t.children[0] == "t" + + assert t.meta.column == 2 + assert t.meta.end_column == 3 + + assert start.column == a.column == 1 + assert start.end_column == a.end_column == 4 + + + def test_expand1(self): g = Lark("""start: a From f14ff6d4d14b500410b8d0d5e14fd2908be95dd9 Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Mon, 5 Jul 2021 14:33:28 +0300 Subject: [PATCH 4/4] Fixed tests to use meta (Tree.column is deprecated) --- tests/test_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_parser.py b/tests/test_parser.py index 40ed131..8fec82d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -109,8 +109,8 @@ class TestParsers(unittest.TestCase): assert t.meta.column == 2 assert t.meta.end_column == 3 - assert start.column == a.column == 1 - assert start.end_column == a.end_column == 4 + assert start.meta.column == a.meta.column == 1 + assert start.meta.end_column == a.meta.end_column == 4