This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

477 lines
16 KiB

  1. from functools import wraps
  2. from .utils import smart_decorator, combine_alternatives
  3. from .tree import Tree
  4. from .exceptions import VisitError, GrammarError
  5. from .lexer import Token
  6. ###{standalone
  7. from inspect import getmembers, getmro
  8. class Discard(Exception):
  9. """When raising the Discard exception in a transformer callback,
  10. that node is discarded and won't appear in the parent.
  11. """
  12. pass
  13. # Transformers
  14. class _Decoratable:
  15. "Provides support for decorating methods with @v_args"
  16. @classmethod
  17. def _apply_decorator(cls, decorator, **kwargs):
  18. mro = getmro(cls)
  19. assert mro[0] is cls
  20. libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)}
  21. for name, value in getmembers(cls):
  22. # Make sure the function isn't inherited (unless it's overwritten)
  23. if name.startswith('_') or (name in libmembers and name not in cls.__dict__):
  24. continue
  25. if not callable(value):
  26. continue
  27. # Skip if v_args already applied (at the function level)
  28. if hasattr(cls.__dict__[name], 'vargs_applied') or hasattr(value, 'vargs_applied'):
  29. continue
  30. static = isinstance(cls.__dict__[name], (staticmethod, classmethod))
  31. setattr(cls, name, decorator(value, static=static, **kwargs))
  32. return cls
  33. def __class_getitem__(cls, _):
  34. return cls
  35. class Transformer(_Decoratable):
  36. """Transformers visit each node of the tree, and run the appropriate method on it according to the node's data.
  37. Methods are provided by the user via inheritance, and called according to ``tree.data``.
  38. The returned value from each method replaces the node in the tree structure.
  39. Transformers work bottom-up (or depth-first), starting with the leaves and ending at the root of the tree.
  40. Transformers can be used to implement map & reduce patterns. Because nodes are reduced from leaf to root,
  41. at any point the callbacks may assume the children have already been transformed (if applicable).
  42. ``Transformer`` can do anything ``Visitor`` can do, but because it reconstructs the tree,
  43. it is slightly less efficient.
  44. All these classes implement the transformer interface:
  45. - ``Transformer`` - Recursively transforms the tree. This is the one you probably want.
  46. - ``Transformer_InPlace`` - Non-recursive. Changes the tree in-place instead of returning new instances
  47. - ``Transformer_InPlaceRecursive`` - Recursive. Changes the tree in-place instead of returning new instances
  48. Parameters:
  49. visit_tokens (bool, optional): Should the transformer visit tokens in addition to rules.
  50. Setting this to ``False`` is slightly faster. Defaults to ``True``.
  51. (For processing ignored tokens, use the ``lexer_callbacks`` options)
  52. NOTE: A transformer without methods essentially performs a non-memoized partial deepcopy.
  53. """
  54. __visit_tokens__ = True # For backwards compatibility
  55. def __init__(self, visit_tokens=True):
  56. self.__visit_tokens__ = visit_tokens
  57. def _call_userfunc(self, tree, new_children=None):
  58. # Assumes tree is already transformed
  59. children = new_children if new_children is not None else tree.children
  60. try:
  61. f = getattr(self, tree.data)
  62. except AttributeError:
  63. return self.__default__(tree.data, children, tree.meta)
  64. else:
  65. try:
  66. wrapper = getattr(f, 'visit_wrapper', None)
  67. if wrapper is not None:
  68. return f.visit_wrapper(f, tree.data, children, tree.meta)
  69. else:
  70. return f(children)
  71. except (GrammarError, Discard):
  72. raise
  73. except Exception as e:
  74. raise VisitError(tree.data, tree, e)
  75. def _call_userfunc_token(self, token):
  76. try:
  77. f = getattr(self, token.type)
  78. except AttributeError:
  79. return self.__default_token__(token)
  80. else:
  81. try:
  82. return f(token)
  83. except (GrammarError, Discard):
  84. raise
  85. except Exception as e:
  86. raise VisitError(token.type, token, e)
  87. def _transform_children(self, children):
  88. for c in children:
  89. try:
  90. if isinstance(c, Tree):
  91. yield self._transform_tree(c)
  92. elif self.__visit_tokens__ and isinstance(c, Token):
  93. yield self._call_userfunc_token(c)
  94. else:
  95. yield c
  96. except Discard:
  97. pass
  98. def _transform_tree(self, tree):
  99. children = list(self._transform_children(tree.children))
  100. return self._call_userfunc(tree, children)
  101. def transform(self, tree):
  102. "Transform the given tree, and return the final result"
  103. return self._transform_tree(tree)
  104. def __mul__(self, other):
  105. """Chain two transformers together, returning a new transformer.
  106. """
  107. return TransformerChain(self, other)
  108. def __default__(self, data, children, meta):
  109. """Default function that is called if there is no attribute matching ``data``
  110. Can be overridden. Defaults to creating a new copy of the tree node (i.e. ``return Tree(data, children, meta)``)
  111. """
  112. return Tree(data, children, meta)
  113. def __default_token__(self, token):
  114. """Default function that is called if there is no attribute matching ``token.type``
  115. Can be overridden. Defaults to returning the token as-is.
  116. """
  117. return token
  118. class InlineTransformer(Transformer): # XXX Deprecated
  119. def _call_userfunc(self, tree, new_children=None):
  120. # Assumes tree is already transformed
  121. children = new_children if new_children is not None else tree.children
  122. try:
  123. f = getattr(self, tree.data)
  124. except AttributeError:
  125. return self.__default__(tree.data, children, tree.meta)
  126. else:
  127. return f(*children)
  128. class TransformerChain(object):
  129. def __init__(self, *transformers):
  130. self.transformers = transformers
  131. def transform(self, tree):
  132. for t in self.transformers:
  133. tree = t.transform(tree)
  134. return tree
  135. def __mul__(self, other):
  136. return TransformerChain(*self.transformers + (other,))
  137. class Transformer_InPlace(Transformer):
  138. """Same as Transformer, but non-recursive, and changes the tree in-place instead of returning new instances
  139. Useful for huge trees. Conservative in memory.
  140. """
  141. def _transform_tree(self, tree): # Cancel recursion
  142. return self._call_userfunc(tree)
  143. def transform(self, tree):
  144. for subtree in tree.iter_subtrees():
  145. subtree.children = list(self._transform_children(subtree.children))
  146. return self._transform_tree(tree)
  147. class Transformer_NonRecursive(Transformer):
  148. """Same as Transformer but non-recursive.
  149. Like Transformer, it doesn't change the original tree.
  150. Useful for huge trees.
  151. """
  152. def transform(self, tree):
  153. # Tree to postfix
  154. rev_postfix = []
  155. q = [tree]
  156. while q:
  157. t = q.pop()
  158. rev_postfix.append(t)
  159. if isinstance(t, Tree):
  160. q += t.children
  161. # Postfix to tree
  162. stack = []
  163. for x in reversed(rev_postfix):
  164. if isinstance(x, Tree):
  165. size = len(x.children)
  166. if size:
  167. args = stack[-size:]
  168. del stack[-size:]
  169. else:
  170. args = []
  171. stack.append(self._call_userfunc(x, args))
  172. elif self.__visit_tokens__ and isinstance(x, Token):
  173. stack.append(self._call_userfunc_token(x))
  174. else:
  175. stack.append(x)
  176. t ,= stack # We should have only one tree remaining
  177. return t
  178. class Transformer_InPlaceRecursive(Transformer):
  179. "Same as Transformer, recursive, but changes the tree in-place instead of returning new instances"
  180. def _transform_tree(self, tree):
  181. tree.children = list(self._transform_children(tree.children))
  182. return self._call_userfunc(tree)
  183. # Visitors
  184. class VisitorBase:
  185. def _call_userfunc(self, tree):
  186. return getattr(self, tree.data, self.__default__)(tree)
  187. def __default__(self, tree):
  188. """Default function that is called if there is no attribute matching ``tree.data``
  189. Can be overridden. Defaults to doing nothing.
  190. """
  191. return tree
  192. def __class_getitem__(cls, _):
  193. return cls
  194. class Visitor(VisitorBase):
  195. """Tree visitor, non-recursive (can handle huge trees).
  196. Visiting a node calls its methods (provided by the user via inheritance) according to ``tree.data``
  197. """
  198. def visit(self, tree):
  199. "Visits the tree, starting with the leaves and finally the root (bottom-up)"
  200. for subtree in tree.iter_subtrees():
  201. self._call_userfunc(subtree)
  202. return tree
  203. def visit_topdown(self,tree):
  204. "Visit the tree, starting at the root, and ending at the leaves (top-down)"
  205. for subtree in tree.iter_subtrees_topdown():
  206. self._call_userfunc(subtree)
  207. return tree
  208. class Visitor_Recursive(VisitorBase):
  209. """Bottom-up visitor, recursive.
  210. Visiting a node calls its methods (provided by the user via inheritance) according to ``tree.data``
  211. Slightly faster than the non-recursive version.
  212. """
  213. def visit(self, tree):
  214. "Visits the tree, starting with the leaves and finally the root (bottom-up)"
  215. for child in tree.children:
  216. if isinstance(child, Tree):
  217. self.visit(child)
  218. self._call_userfunc(tree)
  219. return tree
  220. def visit_topdown(self,tree):
  221. "Visit the tree, starting at the root, and ending at the leaves (top-down)"
  222. self._call_userfunc(tree)
  223. for child in tree.children:
  224. if isinstance(child, Tree):
  225. self.visit_topdown(child)
  226. return tree
  227. def visit_children_decor(func):
  228. "See Interpreter"
  229. @wraps(func)
  230. def inner(cls, tree):
  231. values = cls.visit_children(tree)
  232. return func(cls, values)
  233. return inner
  234. class Interpreter(_Decoratable):
  235. """Interpreter walks the tree starting at the root.
  236. Visits the tree, starting with the root and finally the leaves (top-down)
  237. For each tree node, it calls its methods (provided by user via inheritance) according to ``tree.data``.
  238. Unlike ``Transformer`` and ``Visitor``, the Interpreter doesn't automatically visit its sub-branches.
  239. The user has to explicitly call ``visit``, ``visit_children``, or use the ``@visit_children_decor``.
  240. This allows the user to implement branching and loops.
  241. """
  242. def visit(self, tree):
  243. f = getattr(self, tree.data)
  244. wrapper = getattr(f, 'visit_wrapper', None)
  245. if wrapper is not None:
  246. return f.visit_wrapper(f, tree.data, tree.children, tree.meta)
  247. else:
  248. return f(tree)
  249. def visit_children(self, tree):
  250. return [self.visit(child) if isinstance(child, Tree) else child
  251. for child in tree.children]
  252. def __getattr__(self, name):
  253. return self.__default__
  254. def __default__(self, tree):
  255. return self.visit_children(tree)
  256. # Decorators
  257. def _apply_decorator(obj, decorator, **kwargs):
  258. try:
  259. _apply = obj._apply_decorator
  260. except AttributeError:
  261. return decorator(obj, **kwargs)
  262. else:
  263. return _apply(decorator, **kwargs)
  264. def _inline_args__func(func):
  265. @wraps(func)
  266. def create_decorator(_f, with_self):
  267. if with_self:
  268. def f(self, children):
  269. return _f(self, *children)
  270. else:
  271. def f(self, children):
  272. return _f(*children)
  273. return f
  274. return smart_decorator(func, create_decorator)
  275. def inline_args(obj): # XXX Deprecated
  276. return _apply_decorator(obj, _inline_args__func)
  277. def _visitor_args_func_dec(func, visit_wrapper=None, static=False):
  278. def create_decorator(_f, with_self):
  279. if with_self:
  280. def f(self, *args, **kwargs):
  281. return _f(self, *args, **kwargs)
  282. else:
  283. def f(self, *args, **kwargs):
  284. return _f(*args, **kwargs)
  285. return f
  286. if static:
  287. f = wraps(func)(create_decorator(func, False))
  288. else:
  289. f = smart_decorator(func, create_decorator)
  290. f.vargs_applied = True
  291. f.visit_wrapper = visit_wrapper
  292. return f
  293. def _vargs_inline(f, _data, children, _meta):
  294. return f(*children)
  295. def _vargs_meta_inline(f, _data, children, meta):
  296. return f(meta, *children)
  297. def _vargs_meta(f, _data, children, meta):
  298. return f(children, meta) # TODO swap these for consistency? Backwards incompatible!
  299. def _vargs_tree(f, data, children, meta):
  300. return f(Tree(data, children, meta))
  301. def v_args(inline=False, meta=False, tree=False, wrapper=None):
  302. """A convenience decorator factory for modifying the behavior of user-supplied visitor methods.
  303. By default, callback methods of transformers/visitors accept one argument - a list of the node's children.
  304. ``v_args`` can modify this behavior. When used on a transformer/visitor class definition,
  305. it applies to all the callback methods inside it.
  306. ``v_args`` can be applied to a single method, or to an entire class. When applied to both,
  307. the options given to the method take precedence.
  308. Parameters:
  309. inline (bool, optional): Children are provided as ``*args`` instead of a list argument (not recommended for very long lists).
  310. meta (bool, optional): Provides two arguments: ``children`` and ``meta`` (instead of just the first)
  311. tree (bool, optional): Provides the entire tree as the argument, instead of the children.
  312. wrapper (function, optional): Provide a function to decorate all methods.
  313. Example:
  314. ::
  315. @v_args(inline=True)
  316. class SolveArith(Transformer):
  317. def add(self, left, right):
  318. return left + right
  319. class ReverseNotation(Transformer_InPlace):
  320. @v_args(tree=True)
  321. def tree_node(self, tree):
  322. tree.children = tree.children[::-1]
  323. """
  324. if tree and (meta or inline):
  325. raise ValueError("Visitor functions cannot combine 'tree' with 'meta' or 'inline'.")
  326. func = None
  327. if meta:
  328. if inline:
  329. func = _vargs_meta_inline
  330. else:
  331. func = _vargs_meta
  332. elif inline:
  333. func = _vargs_inline
  334. elif tree:
  335. func = _vargs_tree
  336. if wrapper is not None:
  337. if func is not None:
  338. raise ValueError("Cannot use 'wrapper' along with 'tree', 'meta' or 'inline'.")
  339. func = wrapper
  340. def _visitor_args_dec(obj):
  341. return _apply_decorator(obj, _visitor_args_func_dec, visit_wrapper=func)
  342. return _visitor_args_dec
  343. ###}
  344. # --- Visitor Utilities ---
  345. class CollapseAmbiguities(Transformer):
  346. """
  347. Transforms a tree that contains any number of _ambig nodes into a list of trees,
  348. each one containing an unambiguous tree.
  349. The length of the resulting list is the product of the length of all _ambig nodes.
  350. Warning: This may quickly explode for highly ambiguous trees.
  351. """
  352. def _ambig(self, options):
  353. return sum(options, [])
  354. def __default__(self, data, children_lists, meta):
  355. return [Tree(data, children, meta) for children in combine_alternatives(children_lists)]
  356. def __default_token__(self, t):
  357. return [t]