This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

559 lines
18 KiB

  1. import os.path
  2. from itertools import chain
  3. import re
  4. import codecs
  5. from ast import literal_eval
  6. from .lexer import Lexer, Token, UnexpectedInput
  7. from .parse_tree_builder import ParseTreeBuilder
  8. from .parser_frontends import LALR
  9. from .parsers.lalr_parser import UnexpectedToken
  10. from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef
  11. from .tree import Tree as T, Transformer, InlineTransformer, Visitor
  12. unicode_escape = codecs.getdecoder('unicode_escape')
  13. __path__ = os.path.dirname(__file__)
  14. IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
  15. _TOKEN_NAMES = {
  16. '.' : 'DOT',
  17. ',' : 'COMMA',
  18. ':' : 'COLON',
  19. ';' : 'SEMICOLON',
  20. '+' : 'PLUS',
  21. '-' : 'MINUS',
  22. '*' : 'STAR',
  23. '/' : 'SLASH',
  24. '\\' : 'BACKSLASH',
  25. '|' : 'VBAR',
  26. '?' : 'QMARK',
  27. '!' : 'BANG',
  28. '@' : 'AT',
  29. '#' : 'HASH',
  30. '$' : 'DOLLAR',
  31. '%' : 'PERCENT',
  32. '^' : 'CIRCUMFLEX',
  33. '&' : 'AMPERSAND',
  34. '_' : 'UNDERSCORE',
  35. '<' : 'LESSTHAN',
  36. '>' : 'MORETHAN',
  37. '=' : 'EQUAL',
  38. '"' : 'DBLQUOTE',
  39. '\'' : 'QUOTE',
  40. '`' : 'BACKQUOTE',
  41. '~' : 'TILDE',
  42. '(' : 'LPAR',
  43. ')' : 'RPAR',
  44. '{' : 'LBRACE',
  45. '}' : 'RBRACE',
  46. '[' : 'LSQB',
  47. ']' : 'RSQB',
  48. '\n' : 'NEWLINE',
  49. '\r\n' : 'CRLF',
  50. '\t' : 'TAB',
  51. ' ' : 'SPACE',
  52. }
  53. # Grammar Parser
  54. TOKENS = {
  55. '_LPAR': r'\(',
  56. '_RPAR': r'\)',
  57. '_LBRA': r'\[',
  58. '_RBRA': r'\]',
  59. 'OP': '[+*?](?![a-z])',
  60. '_COLON': ':',
  61. '_OR': r'\|',
  62. '_DOT': r'\.',
  63. 'RULE': '!?[_?]?[a-z][_a-z0-9]*',
  64. 'TOKEN': '_?[A-Z][_A-Z0-9]*',
  65. 'STRING': r'"(\\"|\\\\|[^"])*?"',
  66. 'REGEXP': r'/(?!/)(\\/|\\\\|[^/])*?/',
  67. '_NL': r'(\r?\n)+\s*',
  68. 'WS': r'[ \t]+',
  69. 'COMMENT': r'//[^\n]*',
  70. '_TO': '->',
  71. '_IGNORE': r'%ignore',
  72. '_IMPORT': r'%import',
  73. }
  74. RULES = {
  75. 'start': ['_list'],
  76. '_list': ['_item', '_list _item'],
  77. '_item': ['rule', 'token', 'statement', '_NL'],
  78. 'rule': ['RULE _COLON expansions _NL'],
  79. 'expansions': ['alias',
  80. 'expansions _OR alias',
  81. 'expansions _NL _OR alias'],
  82. '?alias': ['expansion _TO RULE', 'expansion'],
  83. 'expansion': ['_expansion'],
  84. '_expansion': ['', '_expansion expr'],
  85. '?expr': ['atom',
  86. 'atom OP'],
  87. '?atom': ['_LPAR expansions _RPAR',
  88. 'maybe',
  89. 'name',
  90. 'tokenvalue',
  91. 'range'],
  92. '?name': ['RULE', 'TOKEN'],
  93. 'maybe': ['_LBRA expansions _RBRA'],
  94. 'range': ['STRING _DOT _DOT STRING'],
  95. 'token': ['TOKEN _COLON expansions _NL'],
  96. 'statement': ['ignore', 'import'],
  97. 'ignore': ['_IGNORE expansions _NL'],
  98. 'import': ['_IMPORT import_args _NL',
  99. '_IMPORT import_args _TO TOKEN'],
  100. 'import_args': ['_import_args'],
  101. '_import_args': ['name', '_import_args _DOT name'],
  102. 'tokenvalue': ['REGEXP', 'STRING'],
  103. }
  104. class EBNF_to_BNF(InlineTransformer):
  105. def __init__(self):
  106. self.new_rules = {}
  107. self.rules_by_expr = {}
  108. self.prefix = 'anon'
  109. self.i = 0
  110. def _add_recurse_rule(self, type_, expr):
  111. if expr in self.rules_by_expr:
  112. return self.rules_by_expr[expr]
  113. new_name = '__%s_%s_%d' % (self.prefix, type_, self.i)
  114. self.i += 1
  115. t = Token('RULE', new_name, -1)
  116. self.new_rules[new_name] = T('expansions', [T('expansion', [expr]), T('expansion', [t, expr])])
  117. self.rules_by_expr[expr] = t
  118. return t
  119. def expr(self, rule, op):
  120. if op.value == '?':
  121. return T('expansions', [rule, T('expansion', [])])
  122. elif op.value == '+':
  123. # a : b c+ d
  124. # -->
  125. # a : b _c d
  126. # _c : _c c | c;
  127. return self._add_recurse_rule('plus', rule)
  128. elif op.value == '*':
  129. # a : b c* d
  130. # -->
  131. # a : b _c? d
  132. # _c : _c c | c;
  133. new_name = self._add_recurse_rule('star', rule)
  134. return T('expansions', [new_name, T('expansion', [])])
  135. assert False, op
  136. class SimplifyRule_Visitor(Visitor):
  137. @staticmethod
  138. def _flatten(tree):
  139. while True:
  140. to_expand = [i for i, child in enumerate(tree.children)
  141. if isinstance(child, T) and child.data == tree.data]
  142. if not to_expand:
  143. break
  144. tree.expand_kids_by_index(*to_expand)
  145. def expansion(self, tree):
  146. # rules_list unpacking
  147. # a : b (c|d) e
  148. # -->
  149. # a : b c e | b d e
  150. #
  151. # In AST terms:
  152. # expansion(b, expansions(c, d), e)
  153. # -->
  154. # expansions( expansion(b, c, e), expansion(b, d, e) )
  155. while True:
  156. self._flatten(tree)
  157. for i, child in enumerate(tree.children):
  158. if isinstance(child, T) and child.data == 'expansions':
  159. tree.data = 'expansions'
  160. tree.children = [self.visit(T('expansion', [option if i==j else other
  161. for j, other in enumerate(tree.children)]))
  162. for option in child.children]
  163. break
  164. else:
  165. break
  166. def alias(self, tree):
  167. rule, alias_name = tree.children
  168. if rule.data == 'expansions':
  169. aliases = []
  170. for child in tree.children[0].children:
  171. aliases.append(T('alias', [child, alias_name]))
  172. tree.data = 'expansions'
  173. tree.children = aliases
  174. expansions = _flatten
  175. def dict_update_safe(d1, d2):
  176. for k, v in d2.items():
  177. assert k not in d1
  178. d1[k] = v
  179. class RuleTreeToText(Transformer):
  180. def expansions(self, x):
  181. return x
  182. def expansion(self, symbols):
  183. return [sym.value for sym in symbols], None
  184. def alias(self, x):
  185. (expansion, _alias), alias = x
  186. assert _alias is None, (alias, expansion, '-', _alias)
  187. return expansion, alias.value
  188. class SimplifyTree(InlineTransformer):
  189. def maybe(self, expr):
  190. return T('expr', [expr, Token('OP', '?', -1)])
  191. def tokenmods(self, *args):
  192. if len(args) == 1:
  193. return list(args)
  194. tokenmods, value = args
  195. return tokenmods + [value]
  196. class ExtractAnonTokens(InlineTransformer):
  197. def __init__(self, tokens):
  198. self.tokens = tokens
  199. self.token_set = {td.name for td in self.tokens}
  200. self.str_reverse = {td.pattern.value: td.name for td in tokens if isinstance(td.pattern, PatternStr)}
  201. self.re_reverse = {td.pattern.value: td.name for td in tokens if isinstance(td.pattern, PatternRE)}
  202. self.i = 0
  203. def range(self, start, end):
  204. assert start.type == end.type == 'STRING'
  205. start = start.value[1:-1]
  206. end = end.value[1:-1]
  207. assert len(start) == len(end) == 1
  208. regexp = '/[%s-%s]/' % (start, end)
  209. t = Token('REGEXP', regexp)
  210. return self.tokenvalue(t)
  211. def tokenvalue(self, token):
  212. value = token.value[1:-1]
  213. if token.type == 'STRING':
  214. try:
  215. # If already defined, use the user-defined token name
  216. token_name = self.str_reverse[value]
  217. except KeyError:
  218. # Try to assign an indicative anon-token name, otherwise use a numbered name
  219. try:
  220. token_name = _TOKEN_NAMES[value]
  221. except KeyError:
  222. if value.isalnum() and value[0].isalpha() and ('__'+value.upper()) not in self.token_set:
  223. token_name = value.upper() # This can create name duplications for unidentical tokens
  224. else:
  225. token_name = 'ANONSTR_%d' % self.i
  226. self.i += 1
  227. token_name = '__' + token_name
  228. elif token.type == 'REGEXP':
  229. if value in self.re_reverse: # Kind of a wierd placement
  230. token_name = self.re_reverse[value]
  231. else:
  232. token_name = 'ANONRE_%d' % self.i
  233. self.i += 1
  234. else:
  235. assert False, token
  236. if token_name not in self.token_set:
  237. self.token_set.add(token_name)
  238. if token.type == 'STRING':
  239. assert value not in self.str_reverse
  240. self.str_reverse[value] = token_name
  241. else:
  242. assert value not in self.re_reverse
  243. self.re_reverse[value] = token_name
  244. pattern = _tokenvalue_to_pattern(token)
  245. self.tokens.append(TokenDef(token_name, pattern))
  246. return Token('TOKEN', token_name, -1)
  247. def _tokenvalue_to_pattern(tokenvalue):
  248. v = tokenvalue.value
  249. assert v[0] == v[-1] and v[0] in '"/'
  250. s = literal_eval("u'''%s'''" % v[1:-1])
  251. return { 'STRING': PatternStr,
  252. 'REGEXP': PatternRE }[tokenvalue.type](s)
  253. class TokenTreeToPattern(Transformer):
  254. def tokenvalue(self, tv):
  255. tv ,= tv
  256. return _tokenvalue_to_pattern(tv)
  257. def expansion(self, items):
  258. if len(items) == 1:
  259. return items[0]
  260. return PatternRE(''.join(i.to_regexp() for i in items))
  261. def expansions(self, exps):
  262. if len(exps) == 1:
  263. return exps[0]
  264. return PatternRE('(?:%s)' % ('|'.join(i.to_regexp() for i in exps)))
  265. def range(self, items):
  266. assert all(i.type=='STRING' for i in items)
  267. items = [i[1:-1] for i in items]
  268. start, end = items
  269. assert len(start) == len(end) == 1, (start, end)
  270. return PatternRE('[%s-%s]' % (start, end))
  271. def expr(self, args):
  272. inner, op = args
  273. return PatternRE('(?:%s)%s' % (inner.to_regexp(), op))
  274. class Grammar:
  275. def __init__(self, rule_defs, token_defs, extra):
  276. self.token_defs = token_defs
  277. self.rule_defs = rule_defs
  278. self.extra = extra
  279. def compile(self, lexer=False):
  280. # assert lexer
  281. if not lexer:
  282. self.rule_defs += self.token_defs
  283. self.token_defs = []
  284. for name, tree in self.rule_defs:
  285. for tokenvalue in tree.find_data('tokenvalue'):
  286. value ,= tokenvalue.children
  287. if value.type == 'STRING':
  288. assert value[0] == value[-1] == '"'
  289. if len(value)>3:
  290. tokenvalue.data = 'expansion'
  291. tokenvalue.children = [T('tokenvalue', [Token('STRING', '"%s"'%ch)]) for ch in value[1:-1]]
  292. tokendefs = list(self.token_defs)
  293. # =================
  294. # Compile Tokens
  295. # =================
  296. token_tree_to_pattern = TokenTreeToPattern()
  297. # Convert tokens to strings/regexps
  298. tokens = []
  299. for name, token_tree in tokendefs:
  300. pattern = token_tree_to_pattern.transform(token_tree)
  301. tokens.append(TokenDef(name, pattern) )
  302. # Resolve regexp assignments of the form /..${X}../
  303. # XXX This is deprecated, since you can express most regexps with EBNF
  304. # XXX Also, since this happens after import, it can be a source of bugs
  305. token_dict = {td.name: td.pattern.to_regexp() for td in tokens}
  306. while True:
  307. changed = False
  308. for t in tokens:
  309. if isinstance(t.pattern, PatternRE):
  310. sp = re.split(r'(\$\{%s})' % TOKENS['TOKEN'], t.pattern.value)
  311. if sp:
  312. value = ''.join(token_dict[x[2:-1]] if x.startswith('${') and x.endswith('}') else x
  313. for x in sp)
  314. if value != t.pattern.value:
  315. t.pattern.value = value
  316. changed = True
  317. if not changed:
  318. break
  319. # =================
  320. # Compile Rules
  321. # =================
  322. extract_anon = ExtractAnonTokens(tokens)
  323. ebnf_to_bnf = EBNF_to_BNF()
  324. simplify_rule = SimplifyRule_Visitor()
  325. rule_tree_to_text = RuleTreeToText()
  326. rules = {}
  327. for name, rule_tree in self.rule_defs:
  328. assert name not in rules
  329. tree = extract_anon.transform(rule_tree) # Adds to tokens
  330. rules[name] = ebnf_to_bnf.transform(tree)
  331. dict_update_safe(rules, ebnf_to_bnf.new_rules)
  332. for r in rules.values():
  333. simplify_rule.visit(r)
  334. rules = {origin: rule_tree_to_text.transform(tree) for origin, tree in rules.items()}
  335. return tokens, rules, self.extra
  336. class GrammarRule:
  337. def __init__(self, name, expansions):
  338. self.keep_all_tokens = name.startswith('!')
  339. name = name.lstrip('!')
  340. self.expand1 = name.startswith('?')
  341. name = name.lstrip('?')
  342. self.name = name
  343. self.expansions = expansions
  344. _imported_grammars = {}
  345. def import_grammar(grammar_path):
  346. if grammar_path not in _imported_grammars:
  347. for import_path in IMPORT_PATHS:
  348. with open(os.path.join(import_path, grammar_path)) as f:
  349. text = f.read()
  350. grammar = load_grammar(text)
  351. _imported_grammars[grammar_path] = grammar
  352. return _imported_grammars[grammar_path]
  353. def resolve_token_references(token_defs):
  354. token_dict = dict(token_defs)
  355. assert len(token_dict) == len(token_defs), "Same name defined twice?"
  356. while True:
  357. changed = False
  358. for name, token_tree in token_defs:
  359. for exp in chain(token_tree.find_data('expansion'), token_tree.find_data('expr')):
  360. for i, item in enumerate(exp.children):
  361. if isinstance(item, Token):
  362. if item.type == 'RULE':
  363. raise GrammarError("Rules aren't allowed inside tokens (%s in %s)" % (item, name))
  364. if item.type == 'TOKEN':
  365. exp.children[i] = token_dict[item]
  366. changed = True
  367. if not changed:
  368. break
  369. class GrammarLoader:
  370. def __init__(self):
  371. tokens = [TokenDef(name, PatternRE(value)) for name, value in TOKENS.items()]
  372. d = {r: [(x.split(), None) for x in xs] for r, xs in RULES.items()}
  373. rules, callback = ParseTreeBuilder(T).create_tree_builder(d, None)
  374. lexer_conf = LexerConf(tokens, ['WS', 'COMMENT'], None)
  375. parser_conf = ParserConf(rules, callback, 'start')
  376. self.parser = LALR(lexer_conf, parser_conf)
  377. self.simplify_tree = SimplifyTree()
  378. def load_grammar(self, grammar_text):
  379. # for x in self.parser.lex(grammar_text):
  380. # print (x)
  381. try:
  382. tree = self.simplify_tree.transform( self.parser.parse(grammar_text+'\n') )
  383. except UnexpectedInput as e:
  384. raise GrammarError("Unexpected input %r at line %d column %d" % (e.context, e.line, e.column))
  385. except UnexpectedToken as e:
  386. if '_COLON' in e.expected:
  387. raise GrammarError("Missing colon at line %s column %s" % (e.line, e.column))
  388. elif 'tokenvalue' in e.expected:
  389. raise GrammarError("Expecting a value at line %s column %s" % (e.line, e.column))
  390. elif e.expected == ['_OR']:
  391. raise GrammarError("Newline without starting a new option (Expecting '|') at line %s column %s" % (e.line, e.column))
  392. raise
  393. # Extract grammar items
  394. token_defs = [c.children for c in tree.children if c.data=='token']
  395. rule_defs = [c.children for c in tree.children if c.data=='rule']
  396. statements = [c.children for c in tree.children if c.data=='statement']
  397. assert len(token_defs) + len(rule_defs) + len(statements) == len(tree.children)
  398. token_defs = [(name.value, t) for name, t in token_defs]
  399. # Execute statements
  400. ignore = []
  401. for (stmt,) in statements:
  402. if stmt.data == 'ignore':
  403. expansions ,= stmt.children
  404. ignore.append(expansions)
  405. elif stmt.data == 'import':
  406. dotted_path = stmt.children[0].children
  407. name = stmt.children[1] if len(stmt.children)>1 else dotted_path[-1]
  408. grammar_path = os.path.join(*dotted_path[:-1]) + '.g'
  409. g = import_grammar(grammar_path)
  410. token_tree = dict(g.token_defs)[dotted_path[-1]]
  411. token_defs.append([name.value, token_tree])
  412. else:
  413. assert False, command
  414. # Verify correctness 1
  415. for name, _ in token_defs:
  416. if name.startswith('__'):
  417. raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
  418. # Handle ignore tokens
  419. ignore_names = []
  420. for i, t in enumerate(ignore):
  421. if t.data == 'expansions' and len(t.children) == 1:
  422. x ,= t.children
  423. if x.data == 'expansion' and len(x.children) == 1:
  424. item ,= x.children
  425. if isinstance(item, Token) and item.type == 'TOKEN':
  426. # XXX is this really a wise solution? -- Erez
  427. ignore_names.append(item.value)
  428. continue
  429. name = '__IGNORE_%d'%i
  430. token_defs.append((name, t))
  431. ignore_names.append(name)
  432. # Resolve token references
  433. resolve_token_references(token_defs)
  434. # Verify correctness 2
  435. token_names = set()
  436. for name, _ in token_defs:
  437. if name in token_names:
  438. raise GrammarError("Token '%s' defined more than once" % name)
  439. token_names.add(name)
  440. rules = [GrammarRule(name, x) for name, x in rule_defs]
  441. rule_names = set()
  442. for r in rules:
  443. if r.name.startswith('__'):
  444. raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
  445. if r.name in rule_names:
  446. raise GrammarError("Rule '%s' defined more than once" % r.name)
  447. rule_names.add(r.name)
  448. for r in rules:
  449. used_symbols = {t for x in r.expansions.find_data('expansion')
  450. for t in x.scan_values(lambda t: t.type in ('RULE', 'TOKEN'))}
  451. for sym in used_symbols:
  452. if is_terminal(sym):
  453. if sym not in token_names:
  454. raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, r.name))
  455. else:
  456. if sym not in rule_names:
  457. raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, r.name))
  458. # TODO don't include unused tokens, they can only cause trouble!
  459. return Grammar(rule_defs, token_defs, {'ignore': ignore_names})
  460. load_grammar = GrammarLoader().load_grammar