From 907ccf98663d2553a60923ef0c026738f1544fb1 Mon Sep 17 00:00:00 2001 From: MegaIng Date: Wed, 29 Apr 2020 11:19:07 +0200 Subject: [PATCH 1/4] Added documentation for `__default__` --- docs/visitors.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/visitors.md b/docs/visitors.md index c60c1dc..0238e05 100644 --- a/docs/visitors.md +++ b/docs/visitors.md @@ -12,7 +12,7 @@ Visitors visit each node of the tree, and run the appropriate method on it accor They work bottom-up, starting with the leaves and ending at the root of the tree. -**Example** +**Example:** ```python class IncreaseAllNumbers(Visitor): def number(self, tree): @@ -67,7 +67,7 @@ All these classes implement the transformer interface: By default, transformers only visit rules. `visit_tokens=True` will tell Transformer to visit tokens as well. This is a slightly slower alternative to `lexer_callbacks`, but it's easier to maintain and works for all algorithms (even when there isn't a lexer). -Example: +**Example:** ```python class T(Transformer): @@ -95,7 +95,7 @@ When used on a transformer/visitor class definition, it applies to all the callb - `meta` - Provides two arguments: `children` and `meta` (instead of just the first) - `tree` - Provides the entire tree as the argument, instead of the children. -Examples: +**Examples:** ```python @v_args(inline=True) @@ -105,11 +105,19 @@ class SolveArith(Transformer): class ReverseNotation(Transformer_InPlace): - @v_args(tree=True): + @v_args(tree=True) def tree_node(self, tree): tree.children = tree.children[::-1] ``` +### `__default__` and `__default_token__` +These are the functions that are called on if a function with a corresponding name has not been found. + +- The `__default__` method has the signature `(data, children, meta)`, with `data` being the data attribute of the node. It defaults to reconstruct the Tree + +- The `__default_token__` just takes the `Token` as an argument. It defaults to just return the argument. + + ### Discard When raising the `Discard` exception in a transformer callback, that node is discarded and won't appear in the parent. From 8578d6d77ed10f84627e62486d97c586fc8a0cfb Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sat, 2 May 2020 11:07:32 +0300 Subject: [PATCH 2/4] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 68d02bd..147cb4e 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ See more [examples here](https://github.com/lark-parser/lark/tree/master/example - **LALR(1)** parser - Fast and light, competitive with PLY - Can generate a stand-alone parser - - **CYK** parser, for highly ambiguous grammars (NEW! Courtesy of [ehudt](https://github.com/ehudt)) + - **CYK** parser, for highly ambiguous grammars - **EBNF** grammar - **Unicode** fully supported - **Python 2 & 3** compatible From b18d1098867682d9d32032683d6ab45a1e3c17da Mon Sep 17 00:00:00 2001 From: Blank Spruce <32396809+BlankSpruce@users.noreply.github.com> Date: Sun, 3 May 2020 20:50:50 +0200 Subject: [PATCH 3/4] Rewrite iter_subtrees to more efficient version Using OrderedDict provides these properties: - given subtree is yielded only once since it's stored in OrderedDict only once (even though it may be put there multiple times) so no need to double check if subtree was already seen - order of iteration is preserved as it was previously when subtrees to iterate over were stored in a list --- lark/tree.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/lark/tree.py b/lark/tree.py index ee8dfb7..e2e41d9 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -4,6 +4,7 @@ except ImportError: pass from copy import deepcopy +from collections import OrderedDict ###{standalone @@ -58,25 +59,17 @@ class Tree(object): return hash((self.data, tuple(self.children))) def iter_subtrees(self): - # TODO: Re-write as a more efficient version - - visited = set() - q = [self] - - l = [] - while q: - subtree = q.pop() - l.append( subtree ) - if id(subtree) in visited: - continue # already been here from another branch - visited.add(id(subtree)) - q += [c for c in subtree.children if isinstance(c, Tree)] - - seen = set() - for x in reversed(l): - if id(x) not in seen: - yield x - seen.add(id(x)) + queue = [self] + subtrees = OrderedDict() + for subtree in queue: + if id(subtree) in subtrees: + continue + subtrees[id(subtree)] = subtree + queue += [c for c in reversed(subtree.children) if isinstance(c, Tree)] + + del queue + for subtree in reversed(list(subtrees.values())): + yield subtree def find_pred(self, pred): "Find all nodes where pred(tree) == True" From 4c61aaf1dc09aa46fe0e4248e326bd7cf78fd2b7 Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Mon, 4 May 2020 13:30:47 +0300 Subject: [PATCH 4/4] Small refactoring for iter_subtrees --- lark/tree.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lark/tree.py b/lark/tree.py index e2e41d9..84c996a 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -62,14 +62,12 @@ class Tree(object): queue = [self] subtrees = OrderedDict() for subtree in queue: - if id(subtree) in subtrees: - continue subtrees[id(subtree)] = subtree - queue += [c for c in reversed(subtree.children) if isinstance(c, Tree)] + queue += [c for c in reversed(subtree.children) + if isinstance(c, Tree) and id(c) not in subtrees] del queue - for subtree in reversed(list(subtrees.values())): - yield subtree + return reversed(list(subtrees.values())) def find_pred(self, pred): "Find all nodes where pred(tree) == True"