This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

237 lines
8.2 KiB

  1. ## Lexer Implementation
  2. import re
  3. import sre_parse
  4. from .utils import Str, classify
  5. from .common import is_terminal, PatternStr, PatternRE, TokenDef
  6. class LexError(Exception):
  7. pass
  8. class UnexpectedInput(LexError):
  9. def __init__(self, seq, lex_pos, line, column):
  10. context = seq[lex_pos:lex_pos+5]
  11. message = "No token defined for: '%s' in %r at line %d" % (seq[lex_pos], context, line)
  12. super(UnexpectedInput, self).__init__(message)
  13. self.line = line
  14. self.column = column
  15. self.context = context
  16. class Token(Str):
  17. def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
  18. inst = Str.__new__(cls, value)
  19. inst.type = type_
  20. inst.pos_in_stream = pos_in_stream
  21. inst.value = value
  22. inst.line = line
  23. inst.column = column
  24. return inst
  25. @classmethod
  26. def new_borrow_pos(cls, type_, value, borrow_t):
  27. return cls(type_, value, borrow_t.pos_in_stream, line=borrow_t.line, column=borrow_t.column)
  28. def __repr__(self):
  29. return 'Token(%s, %r)' % (self.type, self.value)
  30. def __deepcopy__(self, memo):
  31. return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)
  32. class Regex:
  33. def __init__(self, pattern, flags=()):
  34. self.pattern = pattern
  35. self.flags = flags
  36. def _regexp_has_newline(r):
  37. return '\n' in r or '\\n' in r or ('(?s)' in r and '.' in r)
  38. def _create_unless_callback(strs):
  39. mres = build_mres(strs, match_whole=True)
  40. def unless_callback(t):
  41. # if t in strs:
  42. # t.type = strs[t]
  43. for mre, type_from_index in mres:
  44. m = mre.match(t.value)
  45. if m:
  46. value = m.group(0)
  47. t.type = type_from_index[m.lastindex]
  48. break
  49. return t
  50. return unless_callback
  51. def _create_unless(tokens):
  52. tokens_by_type = classify(tokens, lambda t: type(t.pattern))
  53. assert len(tokens_by_type) <= 2, tokens_by_type.keys()
  54. embedded_strs = set()
  55. delayed_strs = []
  56. callback = {}
  57. for retok in tokens_by_type.get(PatternRE, []):
  58. unless = [] # {}
  59. for strtok in tokens_by_type.get(PatternStr, []):
  60. s = strtok.pattern.value
  61. m = re.match(retok.pattern.to_regexp(), s)
  62. if m and m.group(0) == s:
  63. if strtok.pattern.flags:
  64. delayed_strs.append(strtok)
  65. embedded_strs.add(strtok.name)
  66. unless.append(strtok)
  67. if unless:
  68. callback[retok.name] = _create_unless_callback(unless)
  69. tokens = [t for t in tokens if t.name not in embedded_strs] + delayed_strs
  70. return tokens, callback
  71. def _build_mres(tokens, max_size, match_whole):
  72. # Python sets an unreasonable group limit (currently 100) in its re module
  73. # Worse, the only way to know we reached it is by catching an AssertionError!
  74. # This function recursively tries less and less groups until it's successful.
  75. postfix = '$' if match_whole else ''
  76. mres = []
  77. while tokens:
  78. try:
  79. mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in tokens[:max_size]))
  80. except AssertionError: # Yes, this is what Python provides us.. :/
  81. return _build_mres(tokens, max_size//2, match_whole)
  82. mres.append((mre, {i:n for n,i in mre.groupindex.items()} ))
  83. tokens = tokens[max_size:]
  84. return mres
  85. def build_mres(tokens, match_whole=False):
  86. return _build_mres(tokens, len(tokens), match_whole)
  87. class Lexer(object):
  88. def __init__(self, tokens, ignore=()):
  89. assert all(isinstance(t, TokenDef) for t in tokens), tokens
  90. self.ignore = ignore
  91. self.newline_char = '\n'
  92. tokens = list(tokens)
  93. # Sanitization
  94. for t in tokens:
  95. try:
  96. re.compile(t.pattern.to_regexp())
  97. except:
  98. raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))
  99. width = sre_parse.parse(t.pattern.to_regexp()).getwidth()
  100. if width[0] == 0:
  101. raise LexError("Lexer does not allow zero-width tokens. (%s: %s)" % (t.name, t.pattern))
  102. token_names = {t.name for t in tokens}
  103. for t in ignore:
  104. if t not in token_names:
  105. raise LexError("Token '%s' was marked to ignore but it is not defined!" % t)
  106. # Init
  107. self.newline_types = [t.name for t in tokens if _regexp_has_newline(t.pattern.to_regexp())]
  108. self.ignore_types = [t for t in ignore]
  109. tokens.sort(key=lambda x:(x.pattern.priority, len(x.pattern.value)), reverse=True)
  110. tokens, self.callback = _create_unless(tokens)
  111. assert all(self.callback.values())
  112. self.tokens = tokens
  113. self.mres = build_mres(tokens)
  114. def lex(self, stream):
  115. lex_pos = 0
  116. line = 1
  117. col_start_pos = 0
  118. newline_types = list(self.newline_types)
  119. ignore_types = list(self.ignore_types)
  120. while True:
  121. for mre, type_from_index in self.mres:
  122. m = mre.match(stream, lex_pos)
  123. if m:
  124. value = m.group(0)
  125. type_ = type_from_index[m.lastindex]
  126. if type_ not in ignore_types:
  127. t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
  128. if t.type in self.callback:
  129. t = self.callback[t.type](t)
  130. yield t
  131. if type_ in newline_types:
  132. newlines = value.count(self.newline_char)
  133. if newlines:
  134. line += newlines
  135. col_start_pos = lex_pos + value.rindex(self.newline_char)
  136. lex_pos += len(value)
  137. break
  138. else:
  139. if lex_pos < len(stream):
  140. raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos)
  141. break
  142. class ContextualLexer:
  143. def __init__(self, tokens, states, ignore=(), always_accept=()):
  144. tokens_by_name = {}
  145. for t in tokens:
  146. assert t.name not in tokens_by_name, t
  147. tokens_by_name[t.name] = t
  148. lexer_by_tokens = {}
  149. self.lexers = {}
  150. for state, accepts in states.items():
  151. key = frozenset(accepts)
  152. try:
  153. lexer = lexer_by_tokens[key]
  154. except KeyError:
  155. accepts = set(accepts) | set(ignore) | set(always_accept)
  156. state_tokens = [tokens_by_name[n] for n in accepts if is_terminal(n) and n!='$end']
  157. lexer = Lexer(state_tokens, ignore=ignore)
  158. lexer_by_tokens[key] = lexer
  159. self.lexers[state] = lexer
  160. self.root_lexer = Lexer(tokens, ignore=ignore)
  161. self.set_parser_state(None) # Needs to be set on the outside
  162. def set_parser_state(self, state):
  163. self.parser_state = state
  164. def lex(self, stream):
  165. lex_pos = 0
  166. line = 1
  167. col_start_pos = 0
  168. newline_types = list(self.root_lexer.newline_types)
  169. ignore_types = list(self.root_lexer.ignore_types)
  170. while True:
  171. lexer = self.lexers[self.parser_state]
  172. for mre, type_from_index in lexer.mres:
  173. m = mre.match(stream, lex_pos)
  174. if m:
  175. value = m.group(0)
  176. type_ = type_from_index[m.lastindex]
  177. if type_ not in ignore_types:
  178. t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
  179. if t.type in lexer.callback:
  180. t = lexer.callback[t.type](t)
  181. yield t
  182. if type_ in newline_types:
  183. newlines = value.count(lexer.newline_char)
  184. if newlines:
  185. line += newlines
  186. col_start_pos = lex_pos + value.rindex(lexer.newline_char)
  187. lex_pos += len(value)
  188. break
  189. else:
  190. if lex_pos < len(stream):
  191. print("Allowed tokens:", lexer.tokens)
  192. raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos)
  193. break