This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

217 lines
7.5 KiB

  1. import re
  2. from functools import partial
  3. from .utils import get_regexp_width, Serialize
  4. from .parsers.grammar_analysis import GrammarAnalyzer
  5. from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token
  6. from .parsers import earley, xearley, cyk
  7. from .parsers.lalr_parser import LALR_Parser
  8. from .grammar import Rule
  9. from .tree import Tree
  10. from .common import LexerConf
  11. ###{standalone
  12. def get_frontend(parser, lexer):
  13. if parser=='lalr':
  14. if lexer is None:
  15. raise ValueError('The LALR parser requires use of a lexer')
  16. elif lexer == 'standard':
  17. return LALR_TraditionalLexer
  18. elif lexer == 'contextual':
  19. return LALR_ContextualLexer
  20. elif issubclass(lexer, Lexer):
  21. return partial(LALR_CustomLexer, lexer)
  22. else:
  23. raise ValueError('Unknown lexer: %s' % lexer)
  24. elif parser=='earley':
  25. if lexer=='standard':
  26. return Earley
  27. elif lexer=='dynamic':
  28. return XEarley
  29. elif lexer=='dynamic_complete':
  30. return XEarley_CompleteLex
  31. elif lexer=='contextual':
  32. raise ValueError('The Earley parser does not support the contextual parser')
  33. else:
  34. raise ValueError('Unknown lexer: %s' % lexer)
  35. elif parser == 'cyk':
  36. if lexer == 'standard':
  37. return CYK
  38. else:
  39. raise ValueError('CYK parser requires using standard parser.')
  40. else:
  41. raise ValueError('Unknown parser: %s' % parser)
  42. class _ParserFrontend(Serialize):
  43. def _parse(self, input, start, *args):
  44. if start is None:
  45. start = self.start
  46. if len(start) > 1:
  47. raise ValueError("Lark initialized with more than 1 possible start rule. Must specify which start rule to parse", start)
  48. start ,= start
  49. return self.parser.parse(input, start, *args)
  50. class WithLexer(_ParserFrontend):
  51. lexer = None
  52. parser = None
  53. lexer_conf = None
  54. start = None
  55. __serialize_fields__ = 'parser', 'lexer_conf', 'start'
  56. __serialize_namespace__ = LexerConf,
  57. def __init__(self, lexer_conf, parser_conf, options=None):
  58. self.lexer_conf = lexer_conf
  59. self.start = parser_conf.start
  60. self.postlex = lexer_conf.postlex
  61. @classmethod
  62. def deserialize(cls, data, memo, callbacks, postlex):
  63. inst = super(WithLexer, cls).deserialize(data, memo)
  64. inst.postlex = postlex
  65. inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks)
  66. inst.init_lexer()
  67. return inst
  68. def _serialize(self, data, memo):
  69. data['parser'] = data['parser'].serialize(memo)
  70. def lex(self, text):
  71. stream = self.lexer.lex(text)
  72. return self.postlex.process(stream) if self.postlex else stream
  73. def parse(self, text, start=None):
  74. token_stream = self.lex(text)
  75. sps = self.lexer.set_parser_state
  76. return self._parse(token_stream, start, *[sps] if sps is not NotImplemented else [])
  77. def init_traditional_lexer(self):
  78. self.lexer = TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks)
  79. class LALR_WithLexer(WithLexer):
  80. def __init__(self, lexer_conf, parser_conf, options=None):
  81. debug = options.debug if options else False
  82. self.parser = LALR_Parser(parser_conf, debug=debug)
  83. WithLexer.__init__(self, lexer_conf, parser_conf, options)
  84. self.init_lexer()
  85. def init_lexer(self):
  86. raise NotImplementedError()
  87. class LALR_TraditionalLexer(LALR_WithLexer):
  88. def init_lexer(self):
  89. self.init_traditional_lexer()
  90. class LALR_ContextualLexer(LALR_WithLexer):
  91. def init_lexer(self):
  92. states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()}
  93. always_accept = self.postlex.always_accept if self.postlex else ()
  94. self.lexer = ContextualLexer(self.lexer_conf.tokens, states,
  95. ignore=self.lexer_conf.ignore,
  96. always_accept=always_accept,
  97. user_callbacks=self.lexer_conf.callbacks)
  98. ###}
  99. class LALR_CustomLexer(LALR_WithLexer):
  100. def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None):
  101. self.lexer = lexer_cls(self.lexer_conf)
  102. debug = options.debug if options else False
  103. self.parser = LALR_Parser(parser_conf, debug=debug)
  104. WithLexer.__init__(self, lexer_conf, parser_conf, options)
  105. def tokenize_text(text):
  106. line = 1
  107. col_start_pos = 0
  108. for i, ch in enumerate(text):
  109. if '\n' in ch:
  110. line += ch.count('\n')
  111. col_start_pos = i + ch.rindex('\n')
  112. yield Token('CHAR', ch, line=line, column=i - col_start_pos)
  113. class Earley(WithLexer):
  114. def __init__(self, lexer_conf, parser_conf, options=None):
  115. WithLexer.__init__(self, lexer_conf, parser_conf, options)
  116. self.init_traditional_lexer()
  117. resolve_ambiguity = options.ambiguity == 'resolve'
  118. self.parser = earley.Parser(parser_conf, self.match, resolve_ambiguity=resolve_ambiguity)
  119. def match(self, term, token):
  120. return term.name == token.type
  121. class XEarley(_ParserFrontend):
  122. def __init__(self, lexer_conf, parser_conf, options=None, **kw):
  123. self.token_by_name = {t.name:t for t in lexer_conf.tokens}
  124. self.start = parser_conf.start
  125. self._prepare_match(lexer_conf)
  126. resolve_ambiguity = options.ambiguity == 'resolve'
  127. self.parser = xearley.Parser(parser_conf,
  128. self.match,
  129. ignore=lexer_conf.ignore,
  130. resolve_ambiguity=resolve_ambiguity,
  131. **kw
  132. )
  133. def match(self, term, text, index=0):
  134. return self.regexps[term.name].match(text, index)
  135. def _prepare_match(self, lexer_conf):
  136. self.regexps = {}
  137. for t in lexer_conf.tokens:
  138. if t.priority != 1:
  139. raise ValueError("Dynamic Earley doesn't support weights on terminals", t, t.priority)
  140. regexp = t.pattern.to_regexp()
  141. try:
  142. width = get_regexp_width(regexp)[0]
  143. except ValueError:
  144. raise ValueError("Bad regexp in token %s: %s" % (t.name, regexp))
  145. else:
  146. if width == 0:
  147. raise ValueError("Dynamic Earley doesn't allow zero-width regexps", t)
  148. self.regexps[t.name] = re.compile(regexp)
  149. def parse(self, text, start):
  150. return self._parse(text, start)
  151. class XEarley_CompleteLex(XEarley):
  152. def __init__(self, *args, **kw):
  153. XEarley.__init__(self, *args, complete_lex=True, **kw)
  154. class CYK(WithLexer):
  155. def __init__(self, lexer_conf, parser_conf, options=None):
  156. WithLexer.__init__(self, lexer_conf, parser_conf, options)
  157. self.init_traditional_lexer()
  158. self._analysis = GrammarAnalyzer(parser_conf)
  159. self.parser = cyk.Parser(parser_conf.rules)
  160. self.callbacks = parser_conf.callbacks
  161. def parse(self, text, start):
  162. tokens = list(self.lex(text))
  163. parse = self._parse(tokens, start)
  164. parse = self._transform(parse)
  165. return parse
  166. def _transform(self, tree):
  167. subtrees = list(tree.iter_subtrees())
  168. for subtree in subtrees:
  169. subtree.children = [self._apply_callback(c) if isinstance(c, Tree) else c for c in subtree.children]
  170. return self._apply_callback(tree)
  171. def _apply_callback(self, tree):
  172. return self.callbacks[tree.rule](tree.children)