This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

622 lines
21 KiB

  1. import os.path
  2. from itertools import chain
  3. import re
  4. from ast import literal_eval
  5. from copy import deepcopy
  6. from .lexer import Token, UnexpectedInput
  7. from .parse_tree_builder import ParseTreeBuilder
  8. from .parser_frontends import LALR
  9. from .parsers.lalr_parser import UnexpectedToken
  10. from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef
  11. from .tree import Tree as T, Transformer, InlineTransformer, Visitor
  12. __path__ = os.path.dirname(__file__)
  13. IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
  14. _TOKEN_NAMES = {
  15. '.' : 'DOT',
  16. ',' : 'COMMA',
  17. ':' : 'COLON',
  18. ';' : 'SEMICOLON',
  19. '+' : 'PLUS',
  20. '-' : 'MINUS',
  21. '*' : 'STAR',
  22. '/' : 'SLASH',
  23. '\\' : 'BACKSLASH',
  24. '|' : 'VBAR',
  25. '?' : 'QMARK',
  26. '!' : 'BANG',
  27. '@' : 'AT',
  28. '#' : 'HASH',
  29. '$' : 'DOLLAR',
  30. '%' : 'PERCENT',
  31. '^' : 'CIRCUMFLEX',
  32. '&' : 'AMPERSAND',
  33. '_' : 'UNDERSCORE',
  34. '<' : 'LESSTHAN',
  35. '>' : 'MORETHAN',
  36. '=' : 'EQUAL',
  37. '"' : 'DBLQUOTE',
  38. '\'' : 'QUOTE',
  39. '`' : 'BACKQUOTE',
  40. '~' : 'TILDE',
  41. '(' : 'LPAR',
  42. ')' : 'RPAR',
  43. '{' : 'LBRACE',
  44. '}' : 'RBRACE',
  45. '[' : 'LSQB',
  46. ']' : 'RSQB',
  47. '\n' : 'NEWLINE',
  48. '\r\n' : 'CRLF',
  49. '\t' : 'TAB',
  50. ' ' : 'SPACE',
  51. }
  52. # Grammar Parser
  53. TOKENS = {
  54. '_LPAR': r'\(',
  55. '_RPAR': r'\)',
  56. '_LBRA': r'\[',
  57. '_RBRA': r'\]',
  58. 'OP': '[+*][?]?|[?](?![a-z])',
  59. '_COLON': ':',
  60. '_OR': r'\|',
  61. '_DOT': r'\.',
  62. 'RULE': '!?[_?]?[a-z][_a-z0-9]*',
  63. 'TOKEN': '_?[A-Z][_A-Z0-9]*',
  64. 'STRING': r'"(\\"|\\\\|[^"])*?"',
  65. 'REGEXP': r'/(?!/)(\\/|\\\\|[^/])*?/',
  66. '_NL': r'(\r?\n)+\s*',
  67. 'WS': r'[ \t]+',
  68. 'COMMENT': r'//[^\n]*',
  69. '_TO': '->',
  70. '_IGNORE': r'%ignore',
  71. '_IMPORT': r'%import',
  72. }
  73. RULES = {
  74. 'start': ['_list'],
  75. '_list': ['_item', '_list _item'],
  76. '_item': ['rule', 'token', 'statement', '_NL'],
  77. 'rule': ['RULE _COLON expansions _NL'],
  78. 'expansions': ['alias',
  79. 'expansions _OR alias',
  80. 'expansions _NL _OR alias'],
  81. '?alias': ['expansion _TO RULE', 'expansion'],
  82. 'expansion': ['_expansion'],
  83. '_expansion': ['', '_expansion expr'],
  84. '?expr': ['atom',
  85. 'atom OP'],
  86. '?atom': ['_LPAR expansions _RPAR',
  87. 'maybe',
  88. 'name',
  89. 'literal',
  90. 'range'],
  91. '?name': ['RULE', 'TOKEN'],
  92. 'maybe': ['_LBRA expansions _RBRA'],
  93. 'range': ['STRING _DOT _DOT STRING'],
  94. 'token': ['TOKEN _COLON expansions _NL'],
  95. 'statement': ['ignore', 'import'],
  96. 'ignore': ['_IGNORE expansions _NL'],
  97. 'import': ['_IMPORT import_args _NL',
  98. '_IMPORT import_args _TO TOKEN'],
  99. 'import_args': ['_import_args'],
  100. '_import_args': ['name', '_import_args _DOT name'],
  101. 'literal': ['REGEXP', 'STRING'],
  102. }
  103. class EBNF_to_BNF(InlineTransformer):
  104. def __init__(self):
  105. self.new_rules = {}
  106. self.rules_by_expr = {}
  107. self.prefix = 'anon'
  108. self.i = 0
  109. self.rule_options = None
  110. def _add_recurse_rule(self, type_, expr):
  111. if expr in self.rules_by_expr:
  112. return self.rules_by_expr[expr]
  113. new_name = '__%s_%s_%d' % (self.prefix, type_, self.i)
  114. self.i += 1
  115. t = Token('RULE', new_name, -1)
  116. self.new_rules[new_name] = T('expansions', [T('expansion', [expr]), T('expansion', [t, expr])]), self.rule_options
  117. self.rules_by_expr[expr] = t
  118. return t
  119. def expr(self, rule, op):
  120. if op.value == '?':
  121. return T('expansions', [rule, T('expansion', [])])
  122. elif op.value == '+':
  123. # a : b c+ d
  124. # -->
  125. # a : b _c d
  126. # _c : _c c | c;
  127. return self._add_recurse_rule('plus', rule)
  128. elif op.value == '*':
  129. # a : b c* d
  130. # -->
  131. # a : b _c? d
  132. # _c : _c c | c;
  133. new_name = self._add_recurse_rule('star', rule)
  134. return T('expansions', [new_name, T('expansion', [])])
  135. assert False, op
  136. class SimplifyRule_Visitor(Visitor):
  137. @staticmethod
  138. def _flatten(tree):
  139. while True:
  140. to_expand = [i for i, child in enumerate(tree.children)
  141. if isinstance(child, T) and child.data == tree.data]
  142. if not to_expand:
  143. break
  144. tree.expand_kids_by_index(*to_expand)
  145. def expansion(self, tree):
  146. # rules_list unpacking
  147. # a : b (c|d) e
  148. # -->
  149. # a : b c e | b d e
  150. #
  151. # In AST terms:
  152. # expansion(b, expansions(c, d), e)
  153. # -->
  154. # expansions( expansion(b, c, e), expansion(b, d, e) )
  155. while True:
  156. self._flatten(tree)
  157. for i, child in enumerate(tree.children):
  158. if isinstance(child, T) and child.data == 'expansions':
  159. tree.data = 'expansions'
  160. tree.children = [self.visit(T('expansion', [option if i==j else other
  161. for j, other in enumerate(tree.children)]))
  162. for option in child.children]
  163. break
  164. else:
  165. break
  166. def alias(self, tree):
  167. rule, alias_name = tree.children
  168. if rule.data == 'expansions':
  169. aliases = []
  170. for child in tree.children[0].children:
  171. aliases.append(T('alias', [child, alias_name]))
  172. tree.data = 'expansions'
  173. tree.children = aliases
  174. expansions = _flatten
  175. def dict_update_safe(d1, d2):
  176. for k, v in d2.items():
  177. assert k not in d1
  178. d1[k] = v
  179. class RuleTreeToText(Transformer):
  180. def expansions(self, x):
  181. return x
  182. def expansion(self, symbols):
  183. return [sym.value for sym in symbols], None
  184. def alias(self, x):
  185. (expansion, _alias), alias = x
  186. assert _alias is None, (alias, expansion, '-', _alias)
  187. return expansion, alias.value
  188. class SimplifyTree(InlineTransformer):
  189. def maybe(self, expr):
  190. return T('expr', [expr, Token('OP', '?', -1)])
  191. def tokenmods(self, *args):
  192. if len(args) == 1:
  193. return list(args)
  194. tokenmods, value = args
  195. return tokenmods + [value]
  196. class ExtractAnonTokens(InlineTransformer):
  197. def __init__(self, tokens):
  198. self.tokens = tokens
  199. self.token_set = {td.name for td in self.tokens}
  200. self.token_reverse = {td.pattern: td.name for td in tokens}
  201. self.i = 0
  202. def pattern(self, p):
  203. value = p.value
  204. if isinstance(p, PatternStr):
  205. try:
  206. # If already defined, use the user-defined token name
  207. token_name = self.token_reverse[p]
  208. except KeyError:
  209. # Try to assign an indicative anon-token name, otherwise use a numbered name
  210. try:
  211. token_name = _TOKEN_NAMES[value]
  212. except KeyError:
  213. if value.isalnum() and value[0].isalpha() and ('__'+value.upper()) not in self.token_set:
  214. token_name = '%s%d' % (value.upper(), self.i)
  215. try:
  216. # Make sure we don't have unicode in our token names
  217. token_name.encode('ascii')
  218. except UnicodeEncodeError:
  219. token_name = 'ANONSTR_%d' % self.i
  220. else:
  221. token_name = 'ANONSTR_%d' % self.i
  222. self.i += 1
  223. token_name = '__' + token_name
  224. elif isinstance(p, PatternRE):
  225. if p in self.token_reverse: # Kind of a wierd placement
  226. token_name = self.token_reverse[p]
  227. else:
  228. token_name = 'ANONRE_%d' % self.i
  229. self.i += 1
  230. else:
  231. assert False, p
  232. if token_name not in self.token_set:
  233. assert p not in self.token_reverse
  234. self.token_set.add(token_name)
  235. self.token_reverse[p] = token_name
  236. self.tokens.append(TokenDef(token_name, p))
  237. return Token('TOKEN', token_name, -1)
  238. def _literal_to_pattern(literal):
  239. v = literal.value
  240. assert v[0] == v[-1] and v[0] in '"/'
  241. x = v[1:-1].replace("'", r"\'")
  242. s = literal_eval("u'''%s'''" % x)
  243. return { 'STRING': PatternStr,
  244. 'REGEXP': PatternRE }[literal.type](s)
  245. class PrepareLiterals(InlineTransformer):
  246. def literal(self, literal):
  247. return T('pattern', [_literal_to_pattern(literal)])
  248. def range(self, start, end):
  249. assert start.type == end.type == 'STRING'
  250. start = start.value[1:-1]
  251. end = end.value[1:-1]
  252. assert len(start) == len(end) == 1
  253. regexp = '[%s-%s]' % (start, end)
  254. return T('pattern', [PatternRE(regexp)])
  255. class SplitLiterals(InlineTransformer):
  256. def pattern(self, p):
  257. if isinstance(p, PatternStr) and len(p.value)>1:
  258. return T('expansion', [T('pattern', [PatternStr(ch)]) for ch in p.value])
  259. return T('pattern', [p])
  260. class TokenTreeToPattern(Transformer):
  261. def pattern(self, ps):
  262. p ,= ps
  263. return p
  264. def expansion(self, items):
  265. if len(items) == 1:
  266. return items[0]
  267. return PatternRE(''.join(i.to_regexp() for i in items))
  268. def expansions(self, exps):
  269. if len(exps) == 1:
  270. return exps[0]
  271. return PatternRE('(?:%s)' % ('|'.join(i.to_regexp() for i in exps)))
  272. def expr(self, args):
  273. inner, op = args
  274. return PatternRE('(?:%s)%s' % (inner.to_regexp(), op))
  275. def interleave(l, item):
  276. for e in l:
  277. yield e
  278. if isinstance(e, T):
  279. if e.data == 'literal':
  280. yield item
  281. elif is_terminal(e):
  282. yield item
  283. class Grammar:
  284. def __init__(self, rule_defs, token_defs, extra):
  285. self.token_defs = token_defs
  286. self.rule_defs = rule_defs
  287. self.extra = extra
  288. def compile(self, lexer=False, start=None):
  289. if not lexer:
  290. rule_defs = deepcopy(self.rule_defs)
  291. # XXX VERY HACKY!! There must be a better way..
  292. ignore_tokens = [('_'+name, t) for name, t in self.token_defs if name in self.extra['ignore']]
  293. if ignore_tokens:
  294. self.token_defs = [('_'+name if name in self.extra['ignore'] else name,t) for name,t in self.token_defs]
  295. ignore_names = [t[0] for t in ignore_tokens]
  296. expr = Token('RULE', '__ignore')
  297. for r, tree, _o in rule_defs:
  298. for exp in tree.find_data('expansion'):
  299. exp.children = list(interleave(exp.children, expr))
  300. if r == start:
  301. exp.children = [expr] + exp.children
  302. for exp in tree.find_data('expr'):
  303. exp.children[0] = T('expansion', list(interleave(exp.children[:1], expr)))
  304. x = [T('expansion', [Token('RULE', x)]) for x in ignore_names]
  305. _ignore_tree = T('expr', [T('expansions', x), Token('OP', '?')])
  306. rule_defs.append(('__ignore', _ignore_tree, None))
  307. # End of "ignore" section
  308. rule_defs += [(name, tree, RuleOptions(keep_all_tokens=True)) for name, tree in self.token_defs]
  309. token_defs = []
  310. tokens_to_convert = {name: '__token_'+name for name, tree, _ in rule_defs if is_terminal(name)}
  311. new_rule_defs = []
  312. for name, tree, options in rule_defs:
  313. if name in tokens_to_convert:
  314. if name.startswith('_'):
  315. options = RuleOptions.new_from(options, filter_out=True)
  316. else:
  317. options = RuleOptions.new_from(options, create_token=name)
  318. name = tokens_to_convert[name]
  319. for exp in chain( tree.find_data('expansion'), tree.find_data('expr') ):
  320. for i, sym in enumerate(exp.children):
  321. if sym in tokens_to_convert:
  322. exp.children[i] = Token(sym.type, tokens_to_convert[sym])
  323. new_rule_defs.append((name, tree, options))
  324. rule_defs = new_rule_defs
  325. else:
  326. token_defs = list(self.token_defs)
  327. rule_defs = self.rule_defs
  328. # =================
  329. # Compile Tokens
  330. # =================
  331. token_tree_to_pattern = TokenTreeToPattern()
  332. # Convert tokens to strings/regexps
  333. tokens = []
  334. for name, token_tree in token_defs:
  335. token_tree = PrepareLiterals().transform(token_tree)
  336. pattern = token_tree_to_pattern.transform(token_tree)
  337. tokens.append(TokenDef(name, pattern) )
  338. # Resolve regexp assignments of the form /..${X}../
  339. # XXX This is deprecated, since you can express most regexps with EBNF
  340. # XXX Also, since this happens after import, it can be a source of bugs
  341. token_dict = {td.name: td.pattern.to_regexp() for td in tokens}
  342. while True:
  343. changed = False
  344. for t in tokens:
  345. if isinstance(t.pattern, PatternRE):
  346. sp = re.split(r'(\$\{%s})' % TOKENS['TOKEN'], t.pattern.value)
  347. if sp:
  348. value = ''.join(token_dict[x[2:-1]] if x.startswith('${') and x.endswith('}') else x
  349. for x in sp)
  350. if value != t.pattern.value:
  351. t.pattern.value = value
  352. changed = True
  353. if not changed:
  354. break
  355. # =================
  356. # Compile Rules
  357. # =================
  358. extract_anon = ExtractAnonTokens(tokens)
  359. ebnf_to_bnf = EBNF_to_BNF()
  360. simplify_rule = SimplifyRule_Visitor()
  361. rule_tree_to_text = RuleTreeToText()
  362. rules = {}
  363. for name, rule_tree, options in rule_defs:
  364. assert name not in rules, name
  365. rule_tree = PrepareLiterals().transform(rule_tree)
  366. if not lexer:
  367. rule_tree = SplitLiterals().transform(rule_tree)
  368. tree = extract_anon.transform(rule_tree) # Adds to tokens
  369. ebnf_to_bnf.rule_options = RuleOptions(keep_all_tokens=True) if options and options.keep_all_tokens else None
  370. rules[name] = ebnf_to_bnf.transform(tree), options
  371. dict_update_safe(rules, ebnf_to_bnf.new_rules)
  372. for tree, _o in rules.values():
  373. simplify_rule.visit(tree)
  374. rules = {origin: (rule_tree_to_text.transform(tree), options) for origin, (tree, options) in rules.items()}
  375. return tokens, rules, self.extra
  376. class RuleOptions:
  377. def __init__(self, keep_all_tokens=False, expand1=False, create_token=None, filter_out=False):
  378. self.keep_all_tokens = keep_all_tokens
  379. self.expand1 = expand1
  380. self.create_token = create_token # used for scanless postprocessing
  381. self.filter_out = filter_out # remove this rule from the tree
  382. # used for "token"-rules in scanless
  383. @classmethod
  384. def new_from(cls, options, **kw):
  385. return cls(
  386. keep_all_tokens=options and options.keep_all_tokens,
  387. expand1=options and options.expand1,
  388. **kw)
  389. @classmethod
  390. def from_rule(cls, name, expansions):
  391. keep_all_tokens = name.startswith('!')
  392. name = name.lstrip('!')
  393. expand1 = name.startswith('?')
  394. name = name.lstrip('?')
  395. return name, expansions, cls(keep_all_tokens, expand1)
  396. _imported_grammars = {}
  397. def import_grammar(grammar_path):
  398. if grammar_path not in _imported_grammars:
  399. for import_path in IMPORT_PATHS:
  400. with open(os.path.join(import_path, grammar_path)) as f:
  401. text = f.read()
  402. grammar = load_grammar(text, grammar_path)
  403. _imported_grammars[grammar_path] = grammar
  404. return _imported_grammars[grammar_path]
  405. def resolve_token_references(token_defs):
  406. token_dict = dict(token_defs)
  407. assert len(token_dict) == len(token_defs), "Same name defined twice?"
  408. while True:
  409. changed = False
  410. for name, token_tree in token_defs:
  411. for exp in chain(token_tree.find_data('expansion'), token_tree.find_data('expr')):
  412. for i, item in enumerate(exp.children):
  413. if isinstance(item, Token):
  414. if item.type == 'RULE':
  415. raise GrammarError("Rules aren't allowed inside tokens (%s in %s)" % (item, name))
  416. if item.type == 'TOKEN':
  417. exp.children[i] = token_dict[item]
  418. changed = True
  419. if not changed:
  420. break
  421. class GrammarLoader:
  422. def __init__(self):
  423. tokens = [TokenDef(name, PatternRE(value)) for name, value in TOKENS.items()]
  424. rules = [RuleOptions.from_rule(name, x) for name, x in RULES.items()]
  425. d = {r: ([(x.split(), None) for x in xs], o) for r, xs, o in rules}
  426. rules, callback = ParseTreeBuilder(T).create_tree_builder(d, None)
  427. lexer_conf = LexerConf(tokens, ['WS', 'COMMENT'], None)
  428. parser_conf = ParserConf(rules, callback, 'start')
  429. self.parser = LALR(lexer_conf, parser_conf)
  430. self.simplify_tree = SimplifyTree()
  431. def load_grammar(self, grammar_text, name='<?>'):
  432. try:
  433. tree = self.simplify_tree.transform( self.parser.parse(grammar_text+'\n') )
  434. except UnexpectedInput as e:
  435. raise GrammarError("Unexpected input %r at line %d column %d in %s" % (e.context, e.line, e.column, name))
  436. except UnexpectedToken as e:
  437. if '_COLON' in e.expected:
  438. raise GrammarError("Missing colon at line %s column %s" % (e.line, e.column))
  439. elif 'literal' in e.expected:
  440. raise GrammarError("Expecting a value at line %s column %s" % (e.line, e.column))
  441. elif e.expected == ['_OR']:
  442. raise GrammarError("Newline without starting a new option (Expecting '|') at line %s column %s" % (e.line, e.column))
  443. raise
  444. # Extract grammar items
  445. token_defs = [c.children for c in tree.children if c.data=='token']
  446. rule_defs = [c.children for c in tree.children if c.data=='rule']
  447. statements = [c.children for c in tree.children if c.data=='statement']
  448. assert len(token_defs) + len(rule_defs) + len(statements) == len(tree.children)
  449. token_defs = [(name.value, t) for name, t in token_defs]
  450. # Execute statements
  451. ignore = []
  452. for (stmt,) in statements:
  453. if stmt.data == 'ignore':
  454. expansions ,= stmt.children
  455. ignore.append(expansions)
  456. elif stmt.data == 'import':
  457. dotted_path = stmt.children[0].children
  458. name = stmt.children[1] if len(stmt.children)>1 else dotted_path[-1]
  459. grammar_path = os.path.join(*dotted_path[:-1]) + '.g'
  460. g = import_grammar(grammar_path)
  461. token_tree = dict(g.token_defs)[dotted_path[-1]]
  462. token_defs.append([name.value, token_tree])
  463. else:
  464. assert False, stmt
  465. # Verify correctness 1
  466. for name, _ in token_defs:
  467. if name.startswith('__'):
  468. raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
  469. # Handle ignore tokens
  470. ignore_names = []
  471. for i, t in enumerate(ignore):
  472. if t.data == 'expansions' and len(t.children) == 1:
  473. x ,= t.children
  474. if x.data == 'expansion' and len(x.children) == 1:
  475. item ,= x.children
  476. if isinstance(item, Token) and item.type == 'TOKEN':
  477. # XXX is this really a wise solution? -- Erez
  478. ignore_names.append(item.value)
  479. continue
  480. name = '__IGNORE_%d'%i
  481. token_defs.append((name, t))
  482. ignore_names.append(name)
  483. # Resolve token references
  484. resolve_token_references(token_defs)
  485. # Verify correctness 2
  486. token_names = set()
  487. for name, _ in token_defs:
  488. if name in token_names:
  489. raise GrammarError("Token '%s' defined more than once" % name)
  490. token_names.add(name)
  491. rules = [RuleOptions.from_rule(name, x) for name, x in rule_defs]
  492. rule_names = set()
  493. for name, _x, _o in rules:
  494. if name.startswith('__'):
  495. raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
  496. if name in rule_names:
  497. raise GrammarError("Rule '%s' defined more than once" % name)
  498. rule_names.add(name)
  499. for name, expansions, _o in rules:
  500. used_symbols = {t for x in expansions.find_data('expansion')
  501. for t in x.scan_values(lambda t: t.type in ('RULE', 'TOKEN'))}
  502. for sym in used_symbols:
  503. if is_terminal(sym):
  504. if sym not in token_names:
  505. raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name))
  506. else:
  507. if sym not in rule_names:
  508. raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name))
  509. # TODO don't include unused tokens, they can only cause trouble!
  510. return Grammar(rules, token_defs, {'ignore': ignore_names})
  511. load_grammar = GrammarLoader().load_grammar