from .exceptions import GrammarError from .lexer import Token from .tree import Tree from .visitors import InlineTransformer # XXX Deprecated ###{standalone from functools import partial, wraps class ExpandSingleChild: def __init__(self, node_builder): self.node_builder = node_builder def __call__(self, children): if len(children) == 1: return children[0] else: return self.node_builder(children) class PropagatePositions: def __init__(self, node_builder): self.node_builder = node_builder def __call__(self, children): res = self.node_builder(children) if isinstance(res, Tree) and getattr(res.meta, 'empty', True): res.meta.empty = True for c in children: if isinstance(c, Tree) and c.children and not c.meta.empty: res.meta.line = c.meta.line res.meta.column = c.meta.column res.meta.start_pos = c.meta.start_pos res.meta.empty = False break elif isinstance(c, Token): res.meta.line = c.line res.meta.column = c.column res.meta.start_pos = c.pos_in_stream res.meta.empty = False break for c in reversed(children): if isinstance(c, Tree) and c.children and not c.meta.empty: res.meta.end_line = c.meta.end_line res.meta.end_column = c.meta.end_column res.meta.end_pos = c.meta.end_pos res.meta.empty = False break elif isinstance(c, Token): res.meta.end_line = c.end_line res.meta.end_column = c.end_column res.meta.end_pos = c.pos_in_stream + len(c.value) res.meta.empty = False break return res class ChildFilter: def __init__(self, to_include, append_none, node_builder): self.node_builder = node_builder self.to_include = to_include self.append_none = append_none def __call__(self, children): filtered = [] for i, to_expand, add_none in self.to_include: if add_none: filtered += [None] * add_none if to_expand: filtered += children[i].children else: filtered.append(children[i]) if self.append_none: filtered += [None] * self.append_none return self.node_builder(filtered) class ChildFilterLALR(ChildFilter): "Optimized childfilter for LALR (assumes no duplication in parse tree, so it's safe to change it)" def __call__(self, children): filtered = [] for i, to_expand, add_none in self.to_include: if add_none: filtered += [None] * add_none if to_expand: if filtered: filtered += children[i].children else: # Optimize for left-recursion filtered = children[i].children else: filtered.append(children[i]) if self.append_none: filtered += [None] * self.append_none return self.node_builder(filtered) class ChildFilterLALR_NoPlaceholders(ChildFilter): "Optimized childfilter for LALR (assumes no duplication in parse tree, so it's safe to change it)" def __init__(self, to_include, node_builder): self.node_builder = node_builder self.to_include = to_include def __call__(self, children): filtered = [] for i, to_expand in self.to_include: if to_expand: if filtered: filtered += children[i].children else: # Optimize for left-recursion filtered = children[i].children else: filtered.append(children[i]) return self.node_builder(filtered) def _should_expand(sym): return not sym.is_term and sym.name.startswith('_') def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices): # Prepare empty_indices as: How many Nones to insert at each index? if _empty_indices: assert _empty_indices.count(False) == len(expansion) s = ''.join(str(int(b)) for b in _empty_indices) empty_indices = [len(ones) for ones in s.split('0')] assert len(empty_indices) == len(expansion)+1, (empty_indices, len(expansion)) else: empty_indices = [0] * (len(expansion)+1) to_include = [] nones_to_add = 0 for i, sym in enumerate(expansion): nones_to_add += empty_indices[i] if keep_all_tokens or not (sym.is_term and sym.filter_out): to_include.append((i, _should_expand(sym), nones_to_add)) nones_to_add = 0 nones_to_add += empty_indices[len(expansion)] if _empty_indices or len(to_include) < len(expansion) or any(to_expand for i, to_expand,_ in to_include): if _empty_indices or ambiguous: return partial(ChildFilter if ambiguous else ChildFilterLALR, to_include, nones_to_add) else: # LALR without placeholders return partial(ChildFilterLALR_NoPlaceholders, [(i, x) for i,x,_ in to_include]) class Callback(object): pass def ptb_inline_args(func): @wraps(func) def f(children): return func(*children) return f class ParseTreeBuilder: def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False, maybe_placeholders=False): self.tree_class = tree_class self.propagate_positions = propagate_positions self.always_keep_all_tokens = keep_all_tokens self.ambiguous = ambiguous self.maybe_placeholders = maybe_placeholders self.rule_builders = list(self._init_builders(rules)) self.user_aliases = {} def _init_builders(self, rules): for rule in rules: options = rule.options keep_all_tokens = self.always_keep_all_tokens or (options.keep_all_tokens if options else False) expand_single_child = options.expand1 if options else False wrapper_chain = filter(None, [ (expand_single_child and not rule.alias) and ExpandSingleChild, maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous, options.empty_indices if self.maybe_placeholders and options else None), self.propagate_positions and PropagatePositions, ]) yield rule, wrapper_chain def create_callback(self, transformer=None): callback = Callback() i = 0 for rule, wrapper_chain in self.rule_builders: internal_callback_name = '_cb%d_%s' % (i, rule.origin) i += 1 user_callback_name = rule.alias or rule.origin.name try: f = getattr(transformer, user_callback_name) assert not getattr(f, 'meta', False), "Meta args not supported for internal transformer" # XXX InlineTransformer is deprecated! if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer): f = ptb_inline_args(f) except AttributeError: f = partial(self.tree_class, user_callback_name) self.user_aliases[rule] = rule.alias rule.alias = internal_callback_name for w in wrapper_chain: f = w(f) if hasattr(callback, internal_callback_name): raise GrammarError("Rule '%s' already exists" % (rule,)) setattr(callback, internal_callback_name, f) return callback ###}