This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

245 lines
8.5 KiB

  1. ## Lexer Implementation
  2. import re
  3. from .utils import Str, classify
  4. from .common import is_terminal, PatternStr, PatternRE, TokenDef
  5. class LexError(Exception):
  6. pass
  7. class UnexpectedInput(LexError):
  8. def __init__(self, seq, lex_pos, line, column, allowed=None):
  9. context = seq[lex_pos:lex_pos+5]
  10. message = "No token defined for: '%s' in %r at line %d col %d" % (seq[lex_pos], context, line, column)
  11. super(UnexpectedInput, self).__init__(message)
  12. self.line = line
  13. self.column = column
  14. self.context = context
  15. self.allowed = allowed
  16. class Token(Str):
  17. def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
  18. inst = Str.__new__(cls, value)
  19. inst.type = type_
  20. inst.pos_in_stream = pos_in_stream
  21. inst.value = value
  22. inst.line = line
  23. inst.column = column
  24. return inst
  25. @classmethod
  26. def new_borrow_pos(cls, type_, value, borrow_t):
  27. return cls(type_, value, borrow_t.pos_in_stream, line=borrow_t.line, column=borrow_t.column)
  28. def __repr__(self):
  29. return 'Token(%s, %r)' % (self.type, self.value)
  30. def __deepcopy__(self, memo):
  31. return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)
  32. class Regex:
  33. def __init__(self, pattern, flags=()):
  34. self.pattern = pattern
  35. self.flags = flags
  36. def _regexp_has_newline(r):
  37. return '\n' in r or '\\n' in r or ('(?s)' in r and '.' in r)
  38. def _create_unless_callback(strs):
  39. mres = build_mres(strs, match_whole=True)
  40. def unless_callback(t):
  41. # if t in strs:
  42. # t.type = strs[t]
  43. for mre, type_from_index in mres:
  44. m = mre.match(t.value)
  45. if m:
  46. value = m.group(0)
  47. t.type = type_from_index[m.lastindex]
  48. break
  49. return t
  50. return unless_callback
  51. def _create_unless(tokens):
  52. tokens_by_type = classify(tokens, lambda t: type(t.pattern))
  53. assert len(tokens_by_type) <= 2, tokens_by_type.keys()
  54. embedded_strs = set()
  55. delayed_strs = []
  56. callback = {}
  57. for retok in tokens_by_type.get(PatternRE, []):
  58. unless = [] # {}
  59. for strtok in tokens_by_type.get(PatternStr, []):
  60. s = strtok.pattern.value
  61. m = re.match(retok.pattern.to_regexp(), s)
  62. if m and m.group(0) == s:
  63. if strtok.pattern.flags:
  64. delayed_strs.append(strtok)
  65. embedded_strs.add(strtok.name)
  66. unless.append(strtok)
  67. if unless:
  68. callback[retok.name] = _create_unless_callback(unless)
  69. tokens = [t for t in tokens if t.name not in embedded_strs] + delayed_strs
  70. return tokens, callback
  71. def _build_mres(tokens, max_size, match_whole):
  72. # Python sets an unreasonable group limit (currently 100) in its re module
  73. # Worse, the only way to know we reached it is by catching an AssertionError!
  74. # This function recursively tries less and less groups until it's successful.
  75. postfix = '$' if match_whole else ''
  76. mres = []
  77. while tokens:
  78. try:
  79. mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in tokens[:max_size]))
  80. except AssertionError: # Yes, this is what Python provides us.. :/
  81. return _build_mres(tokens, max_size//2, match_whole)
  82. mres.append((mre, {i:n for n,i in mre.groupindex.items()} ))
  83. tokens = tokens[max_size:]
  84. return mres
  85. def build_mres(tokens, match_whole=False):
  86. return _build_mres(tokens, len(tokens), match_whole)
  87. class Lexer(object):
  88. def __init__(self, tokens, ignore=()):
  89. assert all(isinstance(t, TokenDef) for t in tokens), tokens
  90. self.ignore = ignore
  91. self.newline_char = '\n'
  92. tokens = list(tokens)
  93. # Sanitization
  94. for t in tokens:
  95. try:
  96. re.compile(t.pattern.to_regexp())
  97. except:
  98. raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))
  99. if t.pattern.min_width == 0:
  100. raise LexError("Lexer does not allow zero-width tokens. (%s: %s)" % (t.name, t.pattern))
  101. token_names = {t.name for t in tokens}
  102. for t in ignore:
  103. if t not in token_names:
  104. raise LexError("Token '%s' was marked to ignore but it is not defined!" % t)
  105. # Init
  106. self.newline_types = [t.name for t in tokens if _regexp_has_newline(t.pattern.to_regexp())]
  107. self.ignore_types = [t for t in ignore]
  108. tokens.sort(key=lambda x:(-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name))
  109. tokens, self.callback = _create_unless(tokens)
  110. assert all(self.callback.values())
  111. self.tokens = tokens
  112. self.mres = build_mres(tokens)
  113. def lex(self, stream):
  114. lex_pos = 0
  115. line = 1
  116. col_start_pos = 0
  117. newline_types = list(self.newline_types)
  118. ignore_types = list(self.ignore_types)
  119. while True:
  120. for mre, type_from_index in self.mres:
  121. m = mre.match(stream, lex_pos)
  122. if m:
  123. value = m.group(0)
  124. type_ = type_from_index[m.lastindex]
  125. to_yield = type_ not in ignore_types
  126. if to_yield:
  127. t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
  128. end_col = t.column + len(value)
  129. if t.type in self.callback:
  130. t = self.callback[t.type](t)
  131. if type_ in newline_types:
  132. newlines = value.count(self.newline_char)
  133. if newlines:
  134. line += newlines
  135. last_newline_index = value.rindex(self.newline_char) + 1
  136. col_start_pos = lex_pos + last_newline_index
  137. end_col = len(value) - last_newline_index
  138. if to_yield:
  139. t.end_line = line
  140. t.end_col = end_col
  141. yield t
  142. lex_pos += len(value)
  143. break
  144. else:
  145. if lex_pos < len(stream):
  146. raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos)
  147. break
  148. class ContextualLexer:
  149. def __init__(self, tokens, states, ignore=(), always_accept=()):
  150. tokens_by_name = {}
  151. for t in tokens:
  152. assert t.name not in tokens_by_name, t
  153. tokens_by_name[t.name] = t
  154. lexer_by_tokens = {}
  155. self.lexers = {}
  156. for state, accepts in states.items():
  157. key = frozenset(accepts)
  158. try:
  159. lexer = lexer_by_tokens[key]
  160. except KeyError:
  161. accepts = set(accepts) | set(ignore) | set(always_accept)
  162. state_tokens = [tokens_by_name[n] for n in accepts if is_terminal(n) and n!='$end']
  163. lexer = Lexer(state_tokens, ignore=ignore)
  164. lexer_by_tokens[key] = lexer
  165. self.lexers[state] = lexer
  166. self.root_lexer = Lexer(tokens, ignore=ignore)
  167. self.set_parser_state(None) # Needs to be set on the outside
  168. def set_parser_state(self, state):
  169. self.parser_state = state
  170. def lex(self, stream):
  171. lex_pos = 0
  172. line = 1
  173. col_start_pos = 0
  174. newline_types = list(self.root_lexer.newline_types)
  175. ignore_types = list(self.root_lexer.ignore_types)
  176. while True:
  177. lexer = self.lexers[self.parser_state]
  178. for mre, type_from_index in lexer.mres:
  179. m = mre.match(stream, lex_pos)
  180. if m:
  181. value = m.group(0)
  182. type_ = type_from_index[m.lastindex]
  183. if type_ not in ignore_types:
  184. t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
  185. if t.type in lexer.callback:
  186. t = lexer.callback[t.type](t)
  187. yield t
  188. if type_ in newline_types:
  189. newlines = value.count(lexer.newline_char)
  190. if newlines:
  191. line += newlines
  192. col_start_pos = lex_pos + value.rindex(lexer.newline_char)
  193. lex_pos += len(value)
  194. break
  195. else:
  196. if lex_pos < len(stream):
  197. raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos, lexer.tokens)
  198. break