This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

215 lines
9.9 KiB

  1. """This module implements an scanerless Earley parser.
  2. The core Earley algorithm used here is based on Elizabeth Scott's implementation, here:
  3. https://www.sciencedirect.com/science/article/pii/S1571066108001497
  4. That is probably the best reference for understanding the algorithm here.
  5. The Earley parser outputs an SPPF-tree as per that document. The SPPF tree format
  6. is better documented here:
  7. http://www.bramvandersanden.com/post/2014/06/shared-packed-parse-forest/
  8. """
  9. # Author: Erez Shinan (2017)
  10. # Email : erezshin@gmail.com
  11. from ..visitors import Transformer_InPlace, v_args
  12. from ..exceptions import ParseError, UnexpectedToken
  13. from .grammar_analysis import GrammarAnalyzer
  14. from ..grammar import NonTerminal
  15. from .earley_common import Column, Item
  16. from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode
  17. from collections import deque, defaultdict
  18. class Parser:
  19. def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, forest_sum_visitor = ForestSumVisitor):
  20. analysis = GrammarAnalyzer(parser_conf)
  21. self.parser_conf = parser_conf
  22. self.resolve_ambiguity = resolve_ambiguity
  23. self.FIRST = analysis.FIRST
  24. self.callbacks = {}
  25. self.predictions = {}
  26. ## These could be moved to the grammar analyzer. Pre-computing these is *much* faster than
  27. # the slow 'isupper' in is_terminal.
  28. self.TERMINALS = { sym for r in parser_conf.rules for sym in r.expansion if sym.is_term }
  29. self.NON_TERMINALS = { sym for r in parser_conf.rules for sym in r.expansion if not sym.is_term }
  30. for rule in parser_conf.rules:
  31. self.callbacks[rule] = rule.alias if callable(rule.alias) else getattr(parser_conf.callback, rule.alias)
  32. self.predictions[rule.origin] = [x.rule for x in analysis.expand_rule(rule.origin)]
  33. self.forest_tree_visitor = ForestToTreeVisitor(forest_sum_visitor, self.callbacks)
  34. self.term_matcher = term_matcher
  35. def parse(self, stream, start_symbol=None):
  36. # Define parser functions
  37. start_symbol = NonTerminal(start_symbol or self.parser_conf.start)
  38. match = self.term_matcher
  39. held_completions = defaultdict(list)
  40. node_cache = {}
  41. token_cache = {}
  42. def make_symbol_node(s, start, end):
  43. label = (s, start.i, end.i)
  44. if label in node_cache:
  45. node = node_cache[label]
  46. else:
  47. node = node_cache[label] = SymbolNode(s, start, end)
  48. return node
  49. def predict_and_complete(column, to_scan):
  50. """The core Earley Predictor and Completer.
  51. At each stage of the input, we handling any completed items (things
  52. that matched on the last cycle) and use those to predict what should
  53. come next in the input stream. The completions and any predicted
  54. non-terminals are recursively processed until we reach a set of,
  55. which can be added to the scan list for the next scanner cycle."""
  56. held_completions.clear()
  57. # R (items) = Ei (column.items)
  58. items = deque(column.items)
  59. while items:
  60. item = items.pop() # remove an element, A say, from R
  61. ### The Earley completer
  62. if item.is_complete: ### (item.s == string)
  63. if item.node is None:
  64. item.node = make_symbol_node(item.s, item.start, column)
  65. item.node.add_family(item.s, item.rule, item.start, None, None)
  66. # Empty has 0 length. If we complete an empty symbol in a particular
  67. # parse step, we need to be able to use that same empty symbol to complete
  68. # any predictions that result, that themselves require empty. Avoids
  69. # infinite recursion on empty symbols.
  70. # held_completions is 'H' in E.Scott's paper.
  71. is_empty_item = item.start.i == column.i
  72. if is_empty_item:
  73. held_completions[item.rule.origin] = item.node
  74. originators = [originator for originator in item.start.items if originator.expect is not None and originator.expect == item.s]
  75. for originator in originators:
  76. new_item = originator.advance()
  77. new_item.node = make_symbol_node(new_item.s, originator.start, column)
  78. new_item.node.add_family(new_item.s, new_item.rule, new_item.start, originator.node, item.node)
  79. if new_item.expect in self.TERMINALS:
  80. # Add (B :: aC.B, h, y) to Q
  81. to_scan.add(new_item)
  82. elif new_item not in column.items:
  83. # Add (B :: aC.B, h, y) to Ei and R
  84. column.add(new_item)
  85. items.append(new_item)
  86. ### The Earley predictor
  87. elif item.expect in self.NON_TERMINALS: ### (item.s == lr0)
  88. new_items = []
  89. for rule in self.predictions[item.expect]:
  90. new_item = Item(rule, 0, column)
  91. new_items.append(new_item)
  92. # Process any held completions (H).
  93. if item.expect in held_completions:
  94. new_item = item.advance()
  95. new_item.node = make_symbol_node(new_item.s, item.start, column)
  96. new_item.node.add_family(new_item.s, new_item.rule, new_item.start, item.node, held_completions[item.expect])
  97. new_items.append(new_item)
  98. for new_item in new_items:
  99. if new_item.expect in self.TERMINALS:
  100. to_scan.add(new_item)
  101. elif new_item not in column.items:
  102. column.add(new_item)
  103. items.append(new_item)
  104. def scan(i, token, column, to_scan):
  105. """The core Earley Scanner.
  106. This is a custom implementation of the scanner that uses the
  107. Lark lexer to match tokens. The scan list is built by the
  108. Earley predictor, based on the previously completed tokens.
  109. This ensures that at each phase of the parse we have a custom
  110. lexer context, allowing for more complex ambiguities."""
  111. next_set = Column(i+1, self.FIRST)
  112. next_to_scan = set()
  113. for item in set(to_scan):
  114. if match(item.expect, token):
  115. new_item = item.advance()
  116. new_item.node = make_symbol_node(new_item.s, new_item.start, column)
  117. new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token)
  118. if new_item.expect in self.TERMINALS:
  119. # add (B ::= Aai+1.B, h, y) to Q'
  120. next_to_scan.add(new_item)
  121. else:
  122. # add (B ::= Aa+1.B, h, y) to Ei+1
  123. next_set.add(new_item)
  124. if not next_set and not next_to_scan:
  125. expect = {i.expect.name for i in to_scan}
  126. raise UnexpectedToken(token, expect, considered_rules = set(to_scan))
  127. return next_set, next_to_scan
  128. # Main loop starts
  129. column0 = Column(0, self.FIRST)
  130. column = column0
  131. ## The scan buffer. 'Q' in E.Scott's paper.
  132. to_scan = set()
  133. ## Predict for the start_symbol.
  134. # Add predicted items to the first Earley set (for the predictor) if they
  135. # result in a non-terminal, or the scanner if they result in a terminal.
  136. for rule in self.predictions[start_symbol]:
  137. item = Item(rule, 0, column0)
  138. if item.expect in self.TERMINALS:
  139. to_scan.add(item)
  140. else:
  141. column.add(item)
  142. ## The main Earley loop.
  143. # Run the Prediction/Completion cycle for any Items in the current Earley set.
  144. # Completions will be added to the SPPF tree, and predictions will be recursively
  145. # processed down to terminals/empty nodes to be added to the scanner for the next
  146. # step.
  147. for i, token in enumerate(stream):
  148. predict_and_complete(column, to_scan)
  149. # Clear the node_cache and token_cache, which are only relevant for each
  150. # step in the Earley pass.
  151. node_cache.clear()
  152. token_cache.clear()
  153. column, to_scan = scan(i, token, column, to_scan)
  154. predict_and_complete(column, to_scan)
  155. ## Column is now the final column in the parse. If the parse was successful, the start
  156. # symbol should have been completed in the last step of the Earley cycle, and will be in
  157. # this column. Find the item for the start_symbol, which is the root of the SPPF tree.
  158. solutions = [n.node for n in column.items if n.is_complete and n.node is not None and n.s == start_symbol and n.start is column0]
  159. if not solutions:
  160. raise ParseError('Incomplete parse: Could not find a solution to input')
  161. elif len(solutions) > 1:
  162. raise ParseError('Earley should not generate multiple start symbol items!')
  163. ## If we're not resolving ambiguity, we just return the root of the SPPF tree to the caller.
  164. # This means the caller can work directly with the SPPF tree.
  165. if not self.resolve_ambiguity:
  166. return solutions[0]
  167. # ... otherwise, disambiguate and convert the SPPF to an AST, removing any ambiguities
  168. # according to the rules.
  169. return self.forest_tree_visitor.go(solutions[0])
  170. class ApplyCallbacks(Transformer_InPlace):
  171. def __init__(self, postprocess):
  172. self.postprocess = postprocess
  173. @v_args(meta=True)
  174. def drv(self, children, meta):
  175. return self.postprocess[meta.rule](children)