This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

434 lines
15 KiB

  1. ## Lexer Implementation
  2. import re
  3. from .utils import Str, classify, get_regexp_width, Py36, Serialize
  4. from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken
  5. ###{standalone
  6. from copy import copy
  7. class Pattern(Serialize):
  8. def __init__(self, value, flags=()):
  9. self.value = value
  10. self.flags = frozenset(flags)
  11. def __repr__(self):
  12. return repr(self.to_regexp())
  13. # Pattern Hashing assumes all subclasses have a different priority!
  14. def __hash__(self):
  15. return hash((type(self), self.value, self.flags))
  16. def __eq__(self, other):
  17. return type(self) == type(other) and self.value == other.value and self.flags == other.flags
  18. def to_regexp(self):
  19. raise NotImplementedError()
  20. if Py36:
  21. # Python 3.6 changed syntax for flags in regular expression
  22. def _get_flags(self, value):
  23. for f in self.flags:
  24. value = ('(?%s:%s)' % (f, value))
  25. return value
  26. else:
  27. def _get_flags(self, value):
  28. for f in self.flags:
  29. value = ('(?%s)' % f) + value
  30. return value
  31. class PatternStr(Pattern):
  32. __serialize_fields__ = 'value', 'flags'
  33. type = "str"
  34. def to_regexp(self):
  35. return self._get_flags(re.escape(self.value))
  36. @property
  37. def min_width(self):
  38. return len(self.value)
  39. max_width = min_width
  40. class PatternRE(Pattern):
  41. __serialize_fields__ = 'value', 'flags', '_width'
  42. type = "re"
  43. def to_regexp(self):
  44. return self._get_flags(self.value)
  45. _width = None
  46. def _get_width(self):
  47. if self._width is None:
  48. self._width = get_regexp_width(self.to_regexp())
  49. return self._width
  50. @property
  51. def min_width(self):
  52. return self._get_width()[0]
  53. @property
  54. def max_width(self):
  55. return self._get_width()[1]
  56. class TerminalDef(Serialize):
  57. __serialize_fields__ = 'name', 'pattern', 'priority'
  58. __serialize_namespace__ = PatternStr, PatternRE
  59. def __init__(self, name, pattern, priority=1):
  60. assert isinstance(pattern, Pattern), pattern
  61. self.name = name
  62. self.pattern = pattern
  63. self.priority = priority
  64. def __repr__(self):
  65. return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern)
  66. class Token(Str):
  67. """A string with meta-information, that is produced by the lexer.
  68. When parsing text, the resulting chunks of the input that haven't been discarded,
  69. will end up in the tree as Token instances. The Token class inherits from Python's ``str``,
  70. so normal string comparisons and operations will work as expected.
  71. Attributes:
  72. type: Name of the token (as specified in grammar)
  73. value: Value of the token (redundant, as ``token.value == token`` will always be true)
  74. pos_in_stream: The index of the token in the text
  75. line: The line of the token in the text (starting with 1)
  76. column: The column of the token in the text (starting with 1)
  77. end_line: The line where the token ends
  78. end_column: The next column after the end of the token. For example,
  79. if the token is a single character with a column value of 4,
  80. end_column will be 5.
  81. end_pos: the index where the token ends (basically ``pos_in_stream + len(token)``)
  82. """
  83. __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos')
  84. def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None, end_line=None, end_column=None, end_pos=None):
  85. try:
  86. self = super(Token, cls).__new__(cls, value)
  87. except UnicodeDecodeError:
  88. value = value.decode('latin1')
  89. self = super(Token, cls).__new__(cls, value)
  90. self.type = type_
  91. self.pos_in_stream = pos_in_stream
  92. self.value = value
  93. self.line = line
  94. self.column = column
  95. self.end_line = end_line
  96. self.end_column = end_column
  97. self.end_pos = end_pos
  98. return self
  99. def update(self, type_=None, value=None):
  100. return Token.new_borrow_pos(
  101. type_ if type_ is not None else self.type,
  102. value if value is not None else self.value,
  103. self
  104. )
  105. @classmethod
  106. def new_borrow_pos(cls, type_, value, borrow_t):
  107. return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos)
  108. def __reduce__(self):
  109. return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column, ))
  110. def __repr__(self):
  111. return 'Token(%r, %r)' % (self.type, self.value)
  112. def __deepcopy__(self, memo):
  113. return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)
  114. def __eq__(self, other):
  115. if isinstance(other, Token) and self.type != other.type:
  116. return False
  117. return Str.__eq__(self, other)
  118. __hash__ = Str.__hash__
  119. class LineCounter:
  120. def __init__(self, newline_char):
  121. self.newline_char = newline_char
  122. self.char_pos = 0
  123. self.line = 1
  124. self.column = 1
  125. self.line_start_pos = 0
  126. def feed(self, token, test_newline=True):
  127. """Consume a token and calculate the new line & column.
  128. As an optional optimization, set test_newline=False is token doesn't contain a newline.
  129. """
  130. if test_newline:
  131. newlines = token.count(self.newline_char)
  132. if newlines:
  133. self.line += newlines
  134. self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1
  135. self.char_pos += len(token)
  136. self.column = self.char_pos - self.line_start_pos + 1
  137. class _Lex:
  138. "Built to serve both Lexer and ContextualLexer"
  139. def __init__(self, lexer, state=None):
  140. self.lexer = lexer
  141. self.state = state
  142. def lex(self, stream, newline_types, ignore_types):
  143. newline_types = frozenset(newline_types)
  144. ignore_types = frozenset(ignore_types)
  145. line_ctr = LineCounter('\n' if not self.lexer.use_bytes else b'\n')
  146. last_token = None
  147. while line_ctr.char_pos < len(stream):
  148. lexer = self.lexer
  149. res = lexer.match(stream, line_ctr.char_pos)
  150. if not res:
  151. allowed = {v for m, tfi in lexer.mres for v in tfi.values()} - ignore_types
  152. if not allowed:
  153. allowed = {"<END-OF-FILE>"}
  154. raise UnexpectedCharacters(stream, line_ctr.char_pos, line_ctr.line, line_ctr.column, allowed=allowed, state=self.state, token_history=last_token and [last_token])
  155. value, type_ = res
  156. if type_ not in ignore_types:
  157. t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
  158. line_ctr.feed(value, type_ in newline_types)
  159. t.end_line = line_ctr.line
  160. t.end_column = line_ctr.column
  161. t.end_pos = line_ctr.char_pos
  162. if t.type in lexer.callback:
  163. t = lexer.callback[t.type](t)
  164. if not isinstance(t, Token):
  165. raise ValueError("Callbacks must return a token (returned %r)" % t)
  166. yield t
  167. last_token = t
  168. else:
  169. if type_ in lexer.callback:
  170. t2 = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
  171. lexer.callback[type_](t2)
  172. line_ctr.feed(value, type_ in newline_types)
  173. class UnlessCallback:
  174. def __init__(self, mres):
  175. self.mres = mres
  176. def __call__(self, t):
  177. for mre, type_from_index in self.mres:
  178. m = mre.match(t.value)
  179. if m:
  180. t.type = type_from_index[m.lastindex]
  181. break
  182. return t
  183. class CallChain:
  184. def __init__(self, callback1, callback2, cond):
  185. self.callback1 = callback1
  186. self.callback2 = callback2
  187. self.cond = cond
  188. def __call__(self, t):
  189. t2 = self.callback1(t)
  190. return self.callback2(t) if self.cond(t2) else t2
  191. def _create_unless(terminals, g_regex_flags, re_, use_bytes):
  192. tokens_by_type = classify(terminals, lambda t: type(t.pattern))
  193. assert len(tokens_by_type) <= 2, tokens_by_type.keys()
  194. embedded_strs = set()
  195. callback = {}
  196. for retok in tokens_by_type.get(PatternRE, []):
  197. unless = [] # {}
  198. for strtok in tokens_by_type.get(PatternStr, []):
  199. if strtok.priority > retok.priority:
  200. continue
  201. s = strtok.pattern.value
  202. m = re_.match(retok.pattern.to_regexp(), s, g_regex_flags)
  203. if m and m.group(0) == s:
  204. unless.append(strtok)
  205. if strtok.pattern.flags <= retok.pattern.flags:
  206. embedded_strs.add(strtok)
  207. if unless:
  208. callback[retok.name] = UnlessCallback(build_mres(unless, g_regex_flags, re_, match_whole=True, use_bytes=use_bytes))
  209. terminals = [t for t in terminals if t not in embedded_strs]
  210. return terminals, callback
  211. def _build_mres(terminals, max_size, g_regex_flags, match_whole, re_, use_bytes):
  212. # Python sets an unreasonable group limit (currently 100) in its re module
  213. # Worse, the only way to know we reached it is by catching an AssertionError!
  214. # This function recursively tries less and less groups until it's successful.
  215. postfix = '$' if match_whole else ''
  216. mres = []
  217. while terminals:
  218. pattern = u'|'.join(u'(?P<%s>%s)' % (t.name, t.pattern.to_regexp() + postfix) for t in terminals[:max_size])
  219. if use_bytes:
  220. pattern = pattern.encode('latin-1')
  221. try:
  222. mre = re_.compile(pattern, g_regex_flags)
  223. except AssertionError: # Yes, this is what Python provides us.. :/
  224. return _build_mres(terminals, max_size//2, g_regex_flags, match_whole, re_, use_bytes)
  225. # terms_from_name = {t.name: t for t in terminals[:max_size]}
  226. mres.append((mre, {i:n for n,i in mre.groupindex.items()} ))
  227. terminals = terminals[max_size:]
  228. return mres
  229. def build_mres(terminals, g_regex_flags, re_, use_bytes, match_whole=False):
  230. return _build_mres(terminals, len(terminals), g_regex_flags, match_whole, re_, use_bytes)
  231. def _regexp_has_newline(r):
  232. r"""Expressions that may indicate newlines in a regexp:
  233. - newlines (\n)
  234. - escaped newline (\\n)
  235. - anything but ([^...])
  236. - any-char (.) when the flag (?s) exists
  237. - spaces (\s)
  238. """
  239. return '\n' in r or '\\n' in r or '\\s' in r or '[^' in r or ('(?s' in r and '.' in r)
  240. class Lexer(object):
  241. """Lexer interface
  242. Method Signatures:
  243. lex(self, stream) -> Iterator[Token]
  244. """
  245. lex = NotImplemented
  246. class TraditionalLexer(Lexer):
  247. def __init__(self, conf):
  248. terminals = list(conf.tokens)
  249. assert all(isinstance(t, TerminalDef) for t in terminals), terminals
  250. self.re = conf.re_module
  251. if not conf.skip_validation:
  252. # Sanitization
  253. for t in terminals:
  254. try:
  255. self.re.compile(t.pattern.to_regexp(), conf.g_regex_flags)
  256. except self.re.error:
  257. raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))
  258. if t.pattern.min_width == 0:
  259. raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern))
  260. assert set(conf.ignore) <= {t.name for t in terminals}
  261. # Init
  262. self.newline_types = [t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())]
  263. self.ignore_types = list(conf.ignore)
  264. terminals.sort(key=lambda x:(-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name))
  265. self.terminals = terminals
  266. self.user_callbacks = conf.callbacks
  267. self.g_regex_flags = conf.g_regex_flags
  268. self.use_bytes = conf.use_bytes
  269. self._mres = None
  270. # self.build(g_regex_flags)
  271. def _build(self):
  272. terminals, self.callback = _create_unless(self.terminals, self.g_regex_flags, re_=self.re, use_bytes=self.use_bytes)
  273. assert all(self.callback.values())
  274. for type_, f in self.user_callbacks.items():
  275. if type_ in self.callback:
  276. # Already a callback there, probably UnlessCallback
  277. self.callback[type_] = CallChain(self.callback[type_], f, lambda t: t.type == type_)
  278. else:
  279. self.callback[type_] = f
  280. self._mres = build_mres(terminals, self.g_regex_flags, self.re, self.use_bytes)
  281. @property
  282. def mres(self):
  283. if self._mres is None:
  284. self._build()
  285. return self._mres
  286. def match(self, stream, pos):
  287. for mre, type_from_index in self.mres:
  288. m = mre.match(stream, pos)
  289. if m:
  290. return m.group(0), type_from_index[m.lastindex]
  291. def lex(self, stream):
  292. return _Lex(self).lex(stream, self.newline_types, self.ignore_types)
  293. class ContextualLexer(Lexer):
  294. def __init__(self, conf, states, always_accept=()):
  295. terminals = list(conf.tokens)
  296. tokens_by_name = {}
  297. for t in terminals:
  298. assert t.name not in tokens_by_name, t
  299. tokens_by_name[t.name] = t
  300. trad_conf = copy(conf)
  301. trad_conf.tokens = terminals
  302. lexer_by_tokens = {}
  303. self.lexers = {}
  304. for state, accepts in states.items():
  305. key = frozenset(accepts)
  306. try:
  307. lexer = lexer_by_tokens[key]
  308. except KeyError:
  309. accepts = set(accepts) | set(conf.ignore) | set(always_accept)
  310. state_tokens = [tokens_by_name[n] for n in accepts if n and n in tokens_by_name]
  311. lexer_conf = copy(trad_conf)
  312. lexer_conf.tokens = state_tokens
  313. lexer = TraditionalLexer(lexer_conf)
  314. lexer_by_tokens[key] = lexer
  315. self.lexers[state] = lexer
  316. assert trad_conf.tokens is terminals
  317. self.root_lexer = TraditionalLexer(trad_conf)
  318. def lex(self, stream, get_parser_state):
  319. parser_state = get_parser_state()
  320. l = _Lex(self.lexers[parser_state], parser_state)
  321. try:
  322. for x in l.lex(stream, self.root_lexer.newline_types, self.root_lexer.ignore_types):
  323. yield x
  324. parser_state = get_parser_state()
  325. l.lexer = self.lexers[parser_state]
  326. l.state = parser_state # For debug only, no need to worry about multithreading
  327. except UnexpectedCharacters as e:
  328. # In the contextual lexer, UnexpectedCharacters can mean that the terminal is defined,
  329. # but not in the current context.
  330. # This tests the input against the global context, to provide a nicer error.
  331. root_match = self.root_lexer.match(stream, e.pos_in_stream)
  332. if not root_match:
  333. raise
  334. value, type_ = root_match
  335. t = Token(type_, value, e.pos_in_stream, e.line, e.column)
  336. raise UnexpectedToken(t, e.allowed, state=e.state)
  337. ###}