This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1234 lines
45 KiB

  1. """Parses and creates Grammar objects"""
  2. import hashlib
  3. import os.path
  4. import sys
  5. from collections import namedtuple
  6. from copy import copy, deepcopy
  7. from io import open
  8. import pkgutil
  9. from ast import literal_eval
  10. from numbers import Integral
  11. from .utils import bfs, Py36, logger, classify_bool, is_id_continue, is_id_start, bfs_all_unique
  12. from .lexer import Token, TerminalDef, PatternStr, PatternRE
  13. from .parse_tree_builder import ParseTreeBuilder
  14. from .parser_frontends import ParsingFrontend
  15. from .common import LexerConf, ParserConf
  16. from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
  17. from .utils import classify, suppress, dedup_list, Str
  18. from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError
  19. from .tree import Tree, SlottedTree as ST
  20. from .visitors import Transformer, Visitor, v_args, Transformer_InPlace, Transformer_NonRecursive
  21. inline_args = v_args(inline=True)
  22. __path__ = os.path.dirname(__file__)
  23. IMPORT_PATHS = ['grammars']
  24. EXT = '.lark'
  25. _RE_FLAGS = 'imslux'
  26. _EMPTY = Symbol('__empty__')
  27. _TERMINAL_NAMES = {
  28. '.' : 'DOT',
  29. ',' : 'COMMA',
  30. ':' : 'COLON',
  31. ';' : 'SEMICOLON',
  32. '+' : 'PLUS',
  33. '-' : 'MINUS',
  34. '*' : 'STAR',
  35. '/' : 'SLASH',
  36. '\\' : 'BACKSLASH',
  37. '|' : 'VBAR',
  38. '?' : 'QMARK',
  39. '!' : 'BANG',
  40. '@' : 'AT',
  41. '#' : 'HASH',
  42. '$' : 'DOLLAR',
  43. '%' : 'PERCENT',
  44. '^' : 'CIRCUMFLEX',
  45. '&' : 'AMPERSAND',
  46. '_' : 'UNDERSCORE',
  47. '<' : 'LESSTHAN',
  48. '>' : 'MORETHAN',
  49. '=' : 'EQUAL',
  50. '"' : 'DBLQUOTE',
  51. '\'' : 'QUOTE',
  52. '`' : 'BACKQUOTE',
  53. '~' : 'TILDE',
  54. '(' : 'LPAR',
  55. ')' : 'RPAR',
  56. '{' : 'LBRACE',
  57. '}' : 'RBRACE',
  58. '[' : 'LSQB',
  59. ']' : 'RSQB',
  60. '\n' : 'NEWLINE',
  61. '\r\n' : 'CRLF',
  62. '\t' : 'TAB',
  63. ' ' : 'SPACE',
  64. }
  65. # Grammar Parser
  66. TERMINALS = {
  67. '_LPAR': r'\(',
  68. '_RPAR': r'\)',
  69. '_LBRA': r'\[',
  70. '_RBRA': r'\]',
  71. '_LBRACE': r'\{',
  72. '_RBRACE': r'\}',
  73. 'OP': '[+*]|[?](?![a-z])',
  74. '_COLON': ':',
  75. '_COMMA': ',',
  76. '_OR': r'\|',
  77. '_DOT': r'\.(?!\.)',
  78. '_DOTDOT': r'\.\.',
  79. 'TILDE': '~',
  80. 'RULE': '!?[_?]?[a-z][_a-z0-9]*',
  81. 'TERMINAL': '_?[A-Z][_A-Z0-9]*',
  82. 'STRING': r'"(\\"|\\\\|[^"\n])*?"i?',
  83. 'REGEXP': r'/(?!/)(\\/|\\\\|[^/])*?/[%s]*' % _RE_FLAGS,
  84. '_NL': r'(\r?\n)+\s*',
  85. 'WS': r'[ \t]+',
  86. 'COMMENT': r'\s*//[^\n]*',
  87. '_TO': '->',
  88. '_IGNORE': r'%ignore',
  89. '_OVERRIDE': r'%override',
  90. '_DECLARE': r'%declare',
  91. '_EXTEND': r'%extend',
  92. '_IMPORT': r'%import',
  93. 'NUMBER': r'[+-]?\d+',
  94. }
  95. RULES = {
  96. 'start': ['_list'],
  97. '_list': ['_item', '_list _item'],
  98. '_item': ['rule', 'term', 'ignore', 'import', 'declare', 'override', 'extend', '_NL'],
  99. 'rule': ['RULE template_params _COLON expansions _NL',
  100. 'RULE template_params _DOT NUMBER _COLON expansions _NL'],
  101. 'template_params': ['_LBRACE _template_params _RBRACE',
  102. ''],
  103. '_template_params': ['RULE',
  104. '_template_params _COMMA RULE'],
  105. 'expansions': ['alias',
  106. 'expansions _OR alias',
  107. 'expansions _NL _OR alias'],
  108. '?alias': ['expansion _TO RULE', 'expansion'],
  109. 'expansion': ['_expansion'],
  110. '_expansion': ['', '_expansion expr'],
  111. '?expr': ['atom',
  112. 'atom OP',
  113. 'atom TILDE NUMBER',
  114. 'atom TILDE NUMBER _DOTDOT NUMBER',
  115. ],
  116. '?atom': ['_LPAR expansions _RPAR',
  117. 'maybe',
  118. 'value'],
  119. 'value': ['terminal',
  120. 'nonterminal',
  121. 'literal',
  122. 'range',
  123. 'template_usage'],
  124. 'terminal': ['TERMINAL'],
  125. 'nonterminal': ['RULE'],
  126. '?name': ['RULE', 'TERMINAL'],
  127. 'maybe': ['_LBRA expansions _RBRA'],
  128. 'range': ['STRING _DOTDOT STRING'],
  129. 'template_usage': ['RULE _LBRACE _template_args _RBRACE'],
  130. '_template_args': ['value',
  131. '_template_args _COMMA value'],
  132. 'term': ['TERMINAL _COLON expansions _NL',
  133. 'TERMINAL _DOT NUMBER _COLON expansions _NL'],
  134. 'override': ['_OVERRIDE rule',
  135. '_OVERRIDE term'],
  136. 'extend': ['_EXTEND rule',
  137. '_EXTEND term'],
  138. 'ignore': ['_IGNORE expansions _NL'],
  139. 'declare': ['_DECLARE _declare_args _NL'],
  140. 'import': ['_IMPORT _import_path _NL',
  141. '_IMPORT _import_path _LPAR name_list _RPAR _NL',
  142. '_IMPORT _import_path _TO name _NL'],
  143. '_import_path': ['import_lib', 'import_rel'],
  144. 'import_lib': ['_import_args'],
  145. 'import_rel': ['_DOT _import_args'],
  146. '_import_args': ['name', '_import_args _DOT name'],
  147. 'name_list': ['_name_list'],
  148. '_name_list': ['name', '_name_list _COMMA name'],
  149. '_declare_args': ['name', '_declare_args name'],
  150. 'literal': ['REGEXP', 'STRING'],
  151. }
  152. @inline_args
  153. class EBNF_to_BNF(Transformer_InPlace):
  154. def __init__(self):
  155. self.new_rules = []
  156. self.rules_by_expr = {}
  157. self.prefix = 'anon'
  158. self.i = 0
  159. self.rule_options = None
  160. def _add_recurse_rule(self, type_, expr):
  161. if expr in self.rules_by_expr:
  162. return self.rules_by_expr[expr]
  163. new_name = '__%s_%s_%d' % (self.prefix, type_, self.i)
  164. self.i += 1
  165. t = NonTerminal(new_name)
  166. tree = ST('expansions', [ST('expansion', [expr]), ST('expansion', [t, expr])])
  167. self.new_rules.append((new_name, tree, self.rule_options))
  168. self.rules_by_expr[expr] = t
  169. return t
  170. def expr(self, rule, op, *args):
  171. if op.value == '?':
  172. empty = ST('expansion', [])
  173. return ST('expansions', [rule, empty])
  174. elif op.value == '+':
  175. # a : b c+ d
  176. # -->
  177. # a : b _c d
  178. # _c : _c c | c;
  179. return self._add_recurse_rule('plus', rule)
  180. elif op.value == '*':
  181. # a : b c* d
  182. # -->
  183. # a : b _c? d
  184. # _c : _c c | c;
  185. new_name = self._add_recurse_rule('star', rule)
  186. return ST('expansions', [new_name, ST('expansion', [])])
  187. elif op.value == '~':
  188. if len(args) == 1:
  189. mn = mx = int(args[0])
  190. else:
  191. mn, mx = map(int, args)
  192. if mx < mn or mn < 0:
  193. raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (rule, mn, mx))
  194. return ST('expansions', [ST('expansion', [rule] * n) for n in range(mn, mx+1)])
  195. assert False, op
  196. def maybe(self, rule):
  197. keep_all_tokens = self.rule_options and self.rule_options.keep_all_tokens
  198. def will_not_get_removed(sym):
  199. if isinstance(sym, NonTerminal):
  200. return not sym.name.startswith('_')
  201. if isinstance(sym, Terminal):
  202. return keep_all_tokens or not sym.filter_out
  203. assert False
  204. if any(rule.scan_values(will_not_get_removed)):
  205. empty = _EMPTY
  206. else:
  207. empty = ST('expansion', [])
  208. return ST('expansions', [rule, empty])
  209. class SimplifyRule_Visitor(Visitor):
  210. @staticmethod
  211. def _flatten(tree):
  212. while True:
  213. to_expand = [i for i, child in enumerate(tree.children)
  214. if isinstance(child, Tree) and child.data == tree.data]
  215. if not to_expand:
  216. break
  217. tree.expand_kids_by_index(*to_expand)
  218. def expansion(self, tree):
  219. # rules_list unpacking
  220. # a : b (c|d) e
  221. # -->
  222. # a : b c e | b d e
  223. #
  224. # In AST terms:
  225. # expansion(b, expansions(c, d), e)
  226. # -->
  227. # expansions( expansion(b, c, e), expansion(b, d, e) )
  228. self._flatten(tree)
  229. for i, child in enumerate(tree.children):
  230. if isinstance(child, Tree) and child.data == 'expansions':
  231. tree.data = 'expansions'
  232. tree.children = [self.visit(ST('expansion', [option if i == j else other
  233. for j, other in enumerate(tree.children)]))
  234. for option in dedup_list(child.children)]
  235. self._flatten(tree)
  236. break
  237. def alias(self, tree):
  238. rule, alias_name = tree.children
  239. if rule.data == 'expansions':
  240. aliases = []
  241. for child in tree.children[0].children:
  242. aliases.append(ST('alias', [child, alias_name]))
  243. tree.data = 'expansions'
  244. tree.children = aliases
  245. def expansions(self, tree):
  246. self._flatten(tree)
  247. # Ensure all children are unique
  248. if len(set(tree.children)) != len(tree.children):
  249. tree.children = dedup_list(tree.children) # dedup is expensive, so try to minimize its use
  250. class RuleTreeToText(Transformer):
  251. def expansions(self, x):
  252. return x
  253. def expansion(self, symbols):
  254. return symbols, None
  255. def alias(self, x):
  256. (expansion, _alias), alias = x
  257. assert _alias is None, (alias, expansion, '-', _alias) # Double alias not allowed
  258. return expansion, alias.value
  259. class PrepareAnonTerminals(Transformer_InPlace):
  260. """Create a unique list of anonymous terminals. Attempt to give meaningful names to them when we add them"""
  261. def __init__(self, terminals):
  262. self.terminals = terminals
  263. self.term_set = {td.name for td in self.terminals}
  264. self.term_reverse = {td.pattern: td for td in terminals}
  265. self.i = 0
  266. self.rule_options = None
  267. @inline_args
  268. def pattern(self, p):
  269. value = p.value
  270. if p in self.term_reverse and p.flags != self.term_reverse[p].pattern.flags:
  271. raise GrammarError(u'Conflicting flags for the same terminal: %s' % p)
  272. term_name = None
  273. if isinstance(p, PatternStr):
  274. try:
  275. # If already defined, use the user-defined terminal name
  276. term_name = self.term_reverse[p].name
  277. except KeyError:
  278. # Try to assign an indicative anon-terminal name
  279. try:
  280. term_name = _TERMINAL_NAMES[value]
  281. except KeyError:
  282. if value and is_id_continue(value) and is_id_start(value[0]) and value.upper() not in self.term_set:
  283. term_name = value.upper()
  284. if term_name in self.term_set:
  285. term_name = None
  286. elif isinstance(p, PatternRE):
  287. if p in self.term_reverse: # Kind of a weird placement.name
  288. term_name = self.term_reverse[p].name
  289. else:
  290. assert False, p
  291. if term_name is None:
  292. term_name = '__ANON_%d' % self.i
  293. self.i += 1
  294. if term_name not in self.term_set:
  295. assert p not in self.term_reverse
  296. self.term_set.add(term_name)
  297. termdef = TerminalDef(term_name, p)
  298. self.term_reverse[p] = termdef
  299. self.terminals.append(termdef)
  300. filter_out = False if self.rule_options and self.rule_options.keep_all_tokens else isinstance(p, PatternStr)
  301. return Terminal(term_name, filter_out=filter_out)
  302. class _ReplaceSymbols(Transformer_InPlace):
  303. """Helper for ApplyTemplates"""
  304. def __init__(self):
  305. self.names = {}
  306. def value(self, c):
  307. if len(c) == 1 and isinstance(c[0], Token) and c[0].value in self.names:
  308. return self.names[c[0].value]
  309. return self.__default__('value', c, None)
  310. def template_usage(self, c):
  311. if c[0] in self.names:
  312. return self.__default__('template_usage', [self.names[c[0]].name] + c[1:], None)
  313. return self.__default__('template_usage', c, None)
  314. class ApplyTemplates(Transformer_InPlace):
  315. """Apply the templates, creating new rules that represent the used templates"""
  316. def __init__(self, rule_defs):
  317. self.rule_defs = rule_defs
  318. self.replacer = _ReplaceSymbols()
  319. self.created_templates = set()
  320. def template_usage(self, c):
  321. name = c[0]
  322. args = c[1:]
  323. result_name = "%s{%s}" % (name, ",".join(a.name for a in args))
  324. if result_name not in self.created_templates:
  325. self.created_templates.add(result_name)
  326. (_n, params, tree, options) ,= (t for t in self.rule_defs if t[0] == name)
  327. assert len(params) == len(args), args
  328. result_tree = deepcopy(tree)
  329. self.replacer.names = dict(zip(params, args))
  330. self.replacer.transform(result_tree)
  331. self.rule_defs.append((result_name, [], result_tree, deepcopy(options)))
  332. return NonTerminal(result_name)
  333. def _rfind(s, choices):
  334. return max(s.rfind(c) for c in choices)
  335. def eval_escaping(s):
  336. w = ''
  337. i = iter(s)
  338. for n in i:
  339. w += n
  340. if n == '\\':
  341. try:
  342. n2 = next(i)
  343. except StopIteration:
  344. raise GrammarError("Literal ended unexpectedly (bad escaping): `%r`" % s)
  345. if n2 == '\\':
  346. w += '\\\\'
  347. elif n2 not in 'Uuxnftr':
  348. w += '\\'
  349. w += n2
  350. w = w.replace('\\"', '"').replace("'", "\\'")
  351. to_eval = "u'''%s'''" % w
  352. try:
  353. s = literal_eval(to_eval)
  354. except SyntaxError as e:
  355. raise GrammarError(s, e)
  356. return s
  357. def _literal_to_pattern(literal):
  358. v = literal.value
  359. flag_start = _rfind(v, '/"')+1
  360. assert flag_start > 0
  361. flags = v[flag_start:]
  362. assert all(f in _RE_FLAGS for f in flags), flags
  363. if literal.type == 'STRING' and '\n' in v:
  364. raise GrammarError('You cannot put newlines in string literals')
  365. if literal.type == 'REGEXP' and '\n' in v and 'x' not in flags:
  366. raise GrammarError('You can only use newlines in regular expressions '
  367. 'with the `x` (verbose) flag')
  368. v = v[:flag_start]
  369. assert v[0] == v[-1] and v[0] in '"/'
  370. x = v[1:-1]
  371. s = eval_escaping(x)
  372. if s == "":
  373. raise GrammarError("Empty terminals are not allowed (%s)" % literal)
  374. if literal.type == 'STRING':
  375. s = s.replace('\\\\', '\\')
  376. return PatternStr(s, flags, raw=literal.value)
  377. elif literal.type == 'REGEXP':
  378. return PatternRE(s, flags, raw=literal.value)
  379. else:
  380. assert False, 'Invariant failed: literal.type not in ["STRING", "REGEXP"]'
  381. @inline_args
  382. class PrepareLiterals(Transformer_InPlace):
  383. def literal(self, literal):
  384. return ST('pattern', [_literal_to_pattern(literal)])
  385. def range(self, start, end):
  386. assert start.type == end.type == 'STRING'
  387. start = start.value[1:-1]
  388. end = end.value[1:-1]
  389. assert len(eval_escaping(start)) == len(eval_escaping(end)) == 1
  390. regexp = '[%s-%s]' % (start, end)
  391. return ST('pattern', [PatternRE(regexp)])
  392. def _make_joined_pattern(regexp, flags_set):
  393. # In Python 3.6, a new syntax for flags was introduced, that allows us to restrict the scope
  394. # of flags to a specific regexp group. We are already using it in `lexer.Pattern._get_flags`
  395. # However, for prior Python versions, we still need to use global flags, so we have to make sure
  396. # that there are no flag collisions when we merge several terminals.
  397. flags = ()
  398. if not Py36:
  399. if len(flags_set) > 1:
  400. raise GrammarError("Lark doesn't support joining terminals with conflicting flags in python <3.6!")
  401. elif len(flags_set) == 1:
  402. flags ,= flags_set
  403. return PatternRE(regexp, flags)
  404. class TerminalTreeToPattern(Transformer):
  405. def pattern(self, ps):
  406. p ,= ps
  407. return p
  408. def expansion(self, items):
  409. assert items
  410. if len(items) == 1:
  411. return items[0]
  412. pattern = ''.join(i.to_regexp() for i in items)
  413. return _make_joined_pattern(pattern, {i.flags for i in items})
  414. def expansions(self, exps):
  415. if len(exps) == 1:
  416. return exps[0]
  417. pattern = '(?:%s)' % ('|'.join(i.to_regexp() for i in exps))
  418. return _make_joined_pattern(pattern, {i.flags for i in exps})
  419. def expr(self, args):
  420. inner, op = args[:2]
  421. if op == '~':
  422. if len(args) == 3:
  423. op = "{%d}" % int(args[2])
  424. else:
  425. mn, mx = map(int, args[2:])
  426. if mx < mn:
  427. raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (inner, mn, mx))
  428. op = "{%d,%d}" % (mn, mx)
  429. else:
  430. assert len(args) == 2
  431. return PatternRE('(?:%s)%s' % (inner.to_regexp(), op), inner.flags)
  432. def maybe(self, expr):
  433. return self.expr(expr + ['?'])
  434. def alias(self, t):
  435. raise GrammarError("Aliasing not allowed in terminals (You used -> in the wrong place)")
  436. def value(self, v):
  437. return v[0]
  438. class PrepareSymbols(Transformer_InPlace):
  439. def value(self, v):
  440. v ,= v
  441. if isinstance(v, Tree):
  442. return v
  443. elif v.type == 'RULE':
  444. return NonTerminal(Str(v.value))
  445. elif v.type == 'TERMINAL':
  446. return Terminal(Str(v.value), filter_out=v.startswith('_'))
  447. assert False
  448. def nr_deepcopy_tree(t):
  449. """Deepcopy tree `t` without recursion"""
  450. return Transformer_NonRecursive(False).transform(t)
  451. class Grammar:
  452. def __init__(self, rule_defs, term_defs, ignore):
  453. self.term_defs = term_defs
  454. self.rule_defs = rule_defs
  455. self.ignore = ignore
  456. def compile(self, start, terminals_to_keep):
  457. # We change the trees in-place (to support huge grammars)
  458. # So deepcopy allows calling compile more than once.
  459. term_defs = deepcopy(list(self.term_defs))
  460. rule_defs = [(n,p,nr_deepcopy_tree(t),o) for n,p,t,o in self.rule_defs]
  461. # ===================
  462. # Compile Terminals
  463. # ===================
  464. # Convert terminal-trees to strings/regexps
  465. for name, (term_tree, priority) in term_defs:
  466. if term_tree is None: # Terminal added through %declare
  467. continue
  468. expansions = list(term_tree.find_data('expansion'))
  469. if len(expansions) == 1 and not expansions[0].children:
  470. raise GrammarError("Terminals cannot be empty (%s)" % name)
  471. transformer = PrepareLiterals() * TerminalTreeToPattern()
  472. terminals = [TerminalDef(name, transformer.transform(term_tree), priority)
  473. for name, (term_tree, priority) in term_defs if term_tree]
  474. # =================
  475. # Compile Rules
  476. # =================
  477. # 1. Pre-process terminals
  478. anon_tokens_transf = PrepareAnonTerminals(terminals)
  479. transformer = PrepareLiterals() * PrepareSymbols() * anon_tokens_transf # Adds to terminals
  480. # 2. Inline Templates
  481. transformer *= ApplyTemplates(rule_defs)
  482. # 3. Convert EBNF to BNF (and apply step 1 & 2)
  483. ebnf_to_bnf = EBNF_to_BNF()
  484. rules = []
  485. i = 0
  486. while i < len(rule_defs): # We have to do it like this because rule_defs might grow due to templates
  487. name, params, rule_tree, options = rule_defs[i]
  488. i += 1
  489. if len(params) != 0: # Dont transform templates
  490. continue
  491. rule_options = RuleOptions(keep_all_tokens=True) if options and options.keep_all_tokens else None
  492. ebnf_to_bnf.rule_options = rule_options
  493. ebnf_to_bnf.prefix = name
  494. anon_tokens_transf.rule_options = rule_options
  495. tree = transformer.transform(rule_tree)
  496. res = ebnf_to_bnf.transform(tree)
  497. rules.append((name, res, options))
  498. rules += ebnf_to_bnf.new_rules
  499. assert len(rules) == len({name for name, _t, _o in rules}), "Whoops, name collision"
  500. # 4. Compile tree to Rule objects
  501. rule_tree_to_text = RuleTreeToText()
  502. simplify_rule = SimplifyRule_Visitor()
  503. compiled_rules = []
  504. for rule_content in rules:
  505. name, tree, options = rule_content
  506. simplify_rule.visit(tree)
  507. expansions = rule_tree_to_text.transform(tree)
  508. for i, (expansion, alias) in enumerate(expansions):
  509. if alias and name.startswith('_'):
  510. raise GrammarError("Rule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)"% (name, alias))
  511. empty_indices = [x==_EMPTY for x in expansion]
  512. if any(empty_indices):
  513. exp_options = copy(options) or RuleOptions()
  514. exp_options.empty_indices = empty_indices
  515. expansion = [x for x in expansion if x!=_EMPTY]
  516. else:
  517. exp_options = options
  518. assert all(isinstance(x, Symbol) for x in expansion), expansion
  519. rule = Rule(NonTerminal(name), expansion, i, alias, exp_options)
  520. compiled_rules.append(rule)
  521. # Remove duplicates of empty rules, throw error for non-empty duplicates
  522. if len(set(compiled_rules)) != len(compiled_rules):
  523. duplicates = classify(compiled_rules, lambda x: x)
  524. for dups in duplicates.values():
  525. if len(dups) > 1:
  526. if dups[0].expansion:
  527. raise GrammarError("Rules defined twice: %s\n\n(Might happen due to colliding expansion of optionals: [] or ?)"
  528. % ''.join('\n * %s' % i for i in dups))
  529. # Empty rule; assert all other attributes are equal
  530. assert len({(r.alias, r.order, r.options) for r in dups}) == len(dups)
  531. # Remove duplicates
  532. compiled_rules = list(set(compiled_rules))
  533. # Filter out unused rules
  534. while True:
  535. c = len(compiled_rules)
  536. used_rules = {s for r in compiled_rules
  537. for s in r.expansion
  538. if isinstance(s, NonTerminal)
  539. and s != r.origin}
  540. used_rules |= {NonTerminal(s) for s in start}
  541. compiled_rules, unused = classify_bool(compiled_rules, lambda r: r.origin in used_rules)
  542. for r in unused:
  543. logger.debug("Unused rule: %s", r)
  544. if len(compiled_rules) == c:
  545. break
  546. # Filter out unused terminals
  547. used_terms = {t.name for r in compiled_rules
  548. for t in r.expansion
  549. if isinstance(t, Terminal)}
  550. terminals, unused = classify_bool(terminals, lambda t: t.name in used_terms or t.name in self.ignore or t.name in terminals_to_keep)
  551. if unused:
  552. logger.debug("Unused terminals: %s", [t.name for t in unused])
  553. return terminals, compiled_rules, self.ignore
  554. PackageResource = namedtuple('PackageResource', 'pkg_name path')
  555. class FromPackageLoader(object):
  556. """
  557. Provides a simple way of creating custom import loaders that load from packages via ``pkgutil.get_data`` instead of using `open`.
  558. This allows them to be compatible even from within zip files.
  559. Relative imports are handled, so you can just freely use them.
  560. pkg_name: The name of the package. You can probably provide `__name__` most of the time
  561. search_paths: All the path that will be search on absolute imports.
  562. """
  563. def __init__(self, pkg_name, search_paths=("", )):
  564. self.pkg_name = pkg_name
  565. self.search_paths = search_paths
  566. def __repr__(self):
  567. return "%s(%r, %r)" % (type(self).__name__, self.pkg_name, self.search_paths)
  568. def __call__(self, base_path, grammar_path):
  569. if base_path is None:
  570. to_try = self.search_paths
  571. else:
  572. # Check whether or not the importing grammar was loaded by this module.
  573. if not isinstance(base_path, PackageResource) or base_path.pkg_name != self.pkg_name:
  574. # Technically false, but FileNotFound doesn't exist in python2.7, and this message should never reach the end user anyway
  575. raise IOError()
  576. to_try = [base_path.path]
  577. for path in to_try:
  578. full_path = os.path.join(path, grammar_path)
  579. try:
  580. text = pkgutil.get_data(self.pkg_name, full_path)
  581. except IOError:
  582. continue
  583. else:
  584. return PackageResource(self.pkg_name, full_path), text.decode()
  585. raise IOError()
  586. stdlib_loader = FromPackageLoader('lark', IMPORT_PATHS)
  587. def resolve_term_references(term_dict):
  588. # TODO Solve with transitive closure (maybe)
  589. while True:
  590. changed = False
  591. for name, token_tree in term_dict.items():
  592. if token_tree is None: # Terminal added through %declare
  593. continue
  594. for exp in token_tree.find_data('value'):
  595. item ,= exp.children
  596. if isinstance(item, Token):
  597. if item.type == 'RULE':
  598. raise GrammarError("Rules aren't allowed inside terminals (%s in %s)" % (item, name))
  599. if item.type == 'TERMINAL':
  600. try:
  601. term_value = term_dict[item]
  602. except KeyError:
  603. raise GrammarError("Terminal used but not defined: %s" % item)
  604. assert term_value is not None
  605. exp.children[0] = term_value
  606. changed = True
  607. if not changed:
  608. break
  609. for name, term in term_dict.items():
  610. if term: # Not just declared
  611. for child in term.children:
  612. ids = [id(x) for x in child.iter_subtrees()]
  613. if id(term) in ids:
  614. raise GrammarError("Recursion in terminal '%s' (recursion is only allowed in rules, not terminals)" % name)
  615. def options_from_rule(name, params, *x):
  616. if len(x) > 1:
  617. priority, expansions = x
  618. priority = int(priority)
  619. else:
  620. expansions ,= x
  621. priority = None
  622. params = [t.value for t in params.children] if params is not None else [] # For the grammar parser
  623. keep_all_tokens = name.startswith('!')
  624. name = name.lstrip('!')
  625. expand1 = name.startswith('?')
  626. name = name.lstrip('?')
  627. return name, params, expansions, RuleOptions(keep_all_tokens, expand1, priority=priority,
  628. template_source=(name if params else None))
  629. def symbols_from_strcase(expansion):
  630. return [Terminal(x, filter_out=x.startswith('_')) if x.isupper() else NonTerminal(x) for x in expansion]
  631. @inline_args
  632. class PrepareGrammar(Transformer_InPlace):
  633. def terminal(self, name):
  634. return name
  635. def nonterminal(self, name):
  636. return name
  637. def _find_used_symbols(tree):
  638. assert tree.data == 'expansions'
  639. return {t for x in tree.find_data('expansion')
  640. for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
  641. def _get_parser():
  642. try:
  643. return _get_parser.cache
  644. except AttributeError:
  645. terminals = [TerminalDef(name, PatternRE(value)) for name, value in TERMINALS.items()]
  646. rules = [options_from_rule(name, None, x) for name, x in RULES.items()]
  647. rules = [Rule(NonTerminal(r), symbols_from_strcase(x.split()), i, None, o)
  648. for r, _p, xs, o in rules for i, x in enumerate(xs)]
  649. callback = ParseTreeBuilder(rules, ST).create_callback()
  650. import re
  651. lexer_conf = LexerConf(terminals, re, ['WS', 'COMMENT'])
  652. parser_conf = ParserConf(rules, callback, ['start'])
  653. lexer_conf.lexer_type = 'standard'
  654. parser_conf.parser_type = 'lalr'
  655. _get_parser.cache = ParsingFrontend(lexer_conf, parser_conf, {})
  656. return _get_parser.cache
  657. GRAMMAR_ERRORS = [
  658. ('Incorrect type of value', ['a: 1\n']),
  659. ('Unclosed parenthesis', ['a: (\n']),
  660. ('Unmatched closing parenthesis', ['a: )\n', 'a: [)\n', 'a: (]\n']),
  661. ('Expecting rule or terminal definition (missing colon)', ['a\n', 'A\n', 'a->\n', 'A->\n', 'a A\n']),
  662. ('Illegal name for rules or terminals', ['Aa:\n']),
  663. ('Alias expects lowercase name', ['a: -> "a"\n']),
  664. ('Unexpected colon', ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n']),
  665. ('Misplaced operator', ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n']),
  666. ('Expecting option ("|") or a new rule or terminal definition', ['a:a\n()\n']),
  667. ('Terminal names cannot contain dots', ['A.B\n']),
  668. ('Expecting rule or terminal definition', ['"a"\n']),
  669. ('%import expects a name', ['%import "a"\n']),
  670. ('%ignore expects a value', ['%ignore %import\n']),
  671. ]
  672. def _translate_parser_exception(parse, e):
  673. error = e.match_examples(parse, GRAMMAR_ERRORS, use_accepts=True)
  674. if error:
  675. return error
  676. elif 'STRING' in e.expected:
  677. return "Expecting a value"
  678. def _parse_grammar(text, name, start='start'):
  679. try:
  680. tree = _get_parser().parse(text + '\n', start)
  681. except UnexpectedCharacters as e:
  682. context = e.get_context(text)
  683. raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
  684. (e.line, e.column, name, context))
  685. except UnexpectedToken as e:
  686. context = e.get_context(text)
  687. error = _translate_parser_exception(_get_parser().parse, e)
  688. if error:
  689. raise GrammarError("%s, at line %s column %s\n\n%s" % (error, e.line, e.column, context))
  690. raise
  691. return PrepareGrammar().transform(tree)
  692. def _error_repr(error):
  693. if isinstance(error, UnexpectedToken):
  694. error2 = _translate_parser_exception(_get_parser().parse, error)
  695. if error2:
  696. return error2
  697. expected = ', '.join(error.accepts or error.expected)
  698. return "Unexpected token %r. Expected one of: {%s}" % (str(error.token), expected)
  699. else:
  700. return str(error)
  701. def _search_interactive_parser(interactive_parser, predicate):
  702. def expand(node):
  703. path, p = node
  704. for choice in p.choices():
  705. t = Token(choice, '')
  706. try:
  707. new_p = p.feed_token(t)
  708. except ParseError: # Illegal
  709. pass
  710. else:
  711. yield path + (choice,), new_p
  712. for path, p in bfs_all_unique([((), interactive_parser)], expand):
  713. if predicate(p):
  714. return path, p
  715. def find_grammar_errors(text, start='start'):
  716. errors = []
  717. def on_error(e):
  718. errors.append((e, _error_repr(e)))
  719. # recover to a new line
  720. token_path, _ = _search_interactive_parser(e.interactive_parser.as_immutable(), lambda p: '_NL' in p.choices())
  721. for token_type in token_path:
  722. e.interactive_parser.feed_token(Token(token_type, ''))
  723. e.interactive_parser.feed_token(Token('_NL', '\n'))
  724. return True
  725. _tree = _get_parser().parse(text + '\n', start, on_error=on_error)
  726. errors_by_line = classify(errors, lambda e: e[0].line)
  727. errors = [el[0] for el in errors_by_line.values()] # already sorted
  728. for e in errors:
  729. e[0].interactive_parser = None
  730. return errors
  731. def _get_mangle(prefix, aliases, base_mangle=None):
  732. def mangle(s):
  733. if s in aliases:
  734. s = aliases[s]
  735. else:
  736. if s[0] == '_':
  737. s = '_%s__%s' % (prefix, s[1:])
  738. else:
  739. s = '%s__%s' % (prefix, s)
  740. if base_mangle is not None:
  741. s = base_mangle(s)
  742. return s
  743. return mangle
  744. def _mangle_exp(exp, mangle):
  745. if mangle is None:
  746. return exp
  747. exp = deepcopy(exp) # TODO: is this needed
  748. for t in exp.iter_subtrees():
  749. for i, c in enumerate(t.children):
  750. if isinstance(c, Token) and c.type in ('RULE', 'TERMINAL'):
  751. t.children[i] = Token(c.type, mangle(c.value))
  752. return exp
  753. class GrammarBuilder:
  754. def __init__(self, global_keep_all_tokens=False, import_paths=None, used_files=None):
  755. self.global_keep_all_tokens = global_keep_all_tokens
  756. self.import_paths = import_paths or []
  757. self.used_files = used_files or {}
  758. self._definitions = {}
  759. self._ignore_names = []
  760. def _is_term(self, name):
  761. # Imported terminals are of the form `Path__to__Grammar__file__TERMINAL_NAME`
  762. # Only the last part is the actual name, and the rest might contain mixed case
  763. return name.rpartition('__')[-1].isupper()
  764. def _grammar_error(self, msg, *names):
  765. args = {}
  766. for i, name in enumerate(names, start=1):
  767. postfix = '' if i == 1 else str(i)
  768. args['name' + postfix] = name
  769. args['type' + postfix] = lowercase_type = ("rule", "terminal")[self._is_term(name)]
  770. args['Type' + postfix] = lowercase_type.title()
  771. raise GrammarError(msg.format(**args))
  772. def _check_options(self, name, options):
  773. if self._is_term(name):
  774. if options is None:
  775. options = 1
  776. # if we don't use Integral here, we run into python2.7/python3 problems with long vs int
  777. elif not isinstance(options, Integral):
  778. raise GrammarError("Terminal require a single int as 'options' (e.g. priority), got %s" % (type(options),))
  779. else:
  780. if options is None:
  781. options = RuleOptions()
  782. elif not isinstance(options, RuleOptions):
  783. raise GrammarError("Rules require a RuleOptions instance as 'options'")
  784. if self.global_keep_all_tokens:
  785. options.keep_all_tokens = True
  786. return options
  787. def _define(self, name, exp, params=(), options=None, override=False):
  788. if name in self._definitions:
  789. if not override:
  790. self._grammar_error("{Type} '{name}' defined more than once", name)
  791. elif override:
  792. self._grammar_error("Cannot override a nonexisting {type} {name}", name)
  793. if name.startswith('__'):
  794. self._grammar_error('Names starting with double-underscore are reserved (Error at {name})', name)
  795. self._definitions[name] = (params, exp, self._check_options(name, options))
  796. def _extend(self, name, exp, params=(), options=None):
  797. if name not in self._definitions:
  798. self._grammar_error("Can't extend {type} {name} as it wasn't defined before", name)
  799. if tuple(params) != tuple(self._definitions[name][0]):
  800. self._grammar_error("Cannot extend {type} with different parameters: {name}", name)
  801. # TODO: think about what to do with 'options'
  802. base = self._definitions[name][1]
  803. while len(base.children) == 2:
  804. assert isinstance(base.children[0], Tree) and base.children[0].data == 'expansions', base
  805. base = base.children[0]
  806. base.children.insert(0, exp)
  807. def _ignore(self, exp_or_name):
  808. if isinstance(exp_or_name, str):
  809. self._ignore_names.append(exp_or_name)
  810. else:
  811. assert isinstance(exp_or_name, Tree)
  812. t = exp_or_name
  813. if t.data == 'expansions' and len(t.children) == 1:
  814. t2 ,= t.children
  815. if t2.data=='expansion' and len(t2.children) == 1:
  816. item ,= t2.children
  817. if item.data == 'value':
  818. item ,= item.children
  819. if isinstance(item, Token) and item.type == 'TERMINAL':
  820. self._ignore_names.append(item.value)
  821. return
  822. name = '__IGNORE_%d'% len(self._ignore_names)
  823. self._ignore_names.append(name)
  824. self._definitions[name] = ((), t, 1)
  825. def _declare(self, *names):
  826. for name in names:
  827. self._define(name, None)
  828. def _unpack_import(self, stmt, grammar_name):
  829. if len(stmt.children) > 1:
  830. path_node, arg1 = stmt.children
  831. else:
  832. path_node, = stmt.children
  833. arg1 = None
  834. if isinstance(arg1, Tree): # Multi import
  835. dotted_path = tuple(path_node.children)
  836. names = arg1.children
  837. aliases = dict(zip(names, names)) # Can't have aliased multi import, so all aliases will be the same as names
  838. else: # Single import
  839. dotted_path = tuple(path_node.children[:-1])
  840. if not dotted_path:
  841. name ,= path_node.children
  842. raise GrammarError("Nothing was imported from grammar `%s`" % name)
  843. name = path_node.children[-1] # Get name from dotted path
  844. aliases = {name.value: (arg1 or name).value} # Aliases if exist
  845. if path_node.data == 'import_lib': # Import from library
  846. base_path = None
  847. else: # Relative import
  848. if grammar_name == '<string>': # Import relative to script file path if grammar is coded in script
  849. try:
  850. base_file = os.path.abspath(sys.modules['__main__'].__file__)
  851. except AttributeError:
  852. base_file = None
  853. else:
  854. base_file = grammar_name # Import relative to grammar file path if external grammar file
  855. if base_file:
  856. if isinstance(base_file, PackageResource):
  857. base_path = PackageResource(base_file.pkg_name, os.path.split(base_file.path)[0])
  858. else:
  859. base_path = os.path.split(base_file)[0]
  860. else:
  861. base_path = os.path.abspath(os.path.curdir)
  862. return dotted_path, base_path, aliases
  863. def _unpack_definition(self, tree, mangle):
  864. if tree.data == 'rule':
  865. name, params, exp, opts = options_from_rule(*tree.children)
  866. else:
  867. name = tree.children[0].value
  868. params = () # TODO terminal templates
  869. opts = int(tree.children[1]) if len(tree.children) == 3 else 1 # priority
  870. exp = tree.children[-1]
  871. if mangle is not None:
  872. params = tuple(mangle(p) for p in params)
  873. name = mangle(name)
  874. exp = _mangle_exp(exp, mangle)
  875. return name, exp, params, opts
  876. def load_grammar(self, grammar_text, grammar_name="<?>", mangle=None):
  877. tree = _parse_grammar(grammar_text, grammar_name)
  878. imports = {}
  879. for stmt in tree.children:
  880. if stmt.data == 'import':
  881. dotted_path, base_path, aliases = self._unpack_import(stmt, grammar_name)
  882. try:
  883. import_base_path, import_aliases = imports[dotted_path]
  884. assert base_path == import_base_path, 'Inconsistent base_path for %s.' % '.'.join(dotted_path)
  885. import_aliases.update(aliases)
  886. except KeyError:
  887. imports[dotted_path] = base_path, aliases
  888. for dotted_path, (base_path, aliases) in imports.items():
  889. self.do_import(dotted_path, base_path, aliases, mangle)
  890. for stmt in tree.children:
  891. if stmt.data in ('term', 'rule'):
  892. self._define(*self._unpack_definition(stmt, mangle))
  893. elif stmt.data == 'override':
  894. r ,= stmt.children
  895. self._define(*self._unpack_definition(r, mangle), override=True)
  896. elif stmt.data == 'extend':
  897. r ,= stmt.children
  898. self._extend(*self._unpack_definition(r, mangle))
  899. elif stmt.data == 'ignore':
  900. # if mangle is not None, we shouldn't apply ignore, since we aren't in a toplevel grammar
  901. if mangle is None:
  902. self._ignore(*stmt.children)
  903. elif stmt.data == 'declare':
  904. names = [t.value for t in stmt.children]
  905. if mangle is None:
  906. self._declare(*names)
  907. else:
  908. self._declare(*map(mangle, names))
  909. elif stmt.data == 'import':
  910. pass
  911. else:
  912. assert False, stmt
  913. term_defs = { name: exp
  914. for name, (_params, exp, _options) in self._definitions.items()
  915. if self._is_term(name)
  916. }
  917. resolve_term_references(term_defs)
  918. def _remove_unused(self, used):
  919. def rule_dependencies(symbol):
  920. if self._is_term(symbol):
  921. return []
  922. try:
  923. params, tree,_ = self._definitions[symbol]
  924. except KeyError:
  925. return []
  926. return _find_used_symbols(tree) - set(params)
  927. _used = set(bfs(used, rule_dependencies))
  928. self._definitions = {k: v for k, v in self._definitions.items() if k in _used}
  929. def do_import(self, dotted_path, base_path, aliases, base_mangle=None):
  930. assert dotted_path
  931. mangle = _get_mangle('__'.join(dotted_path), aliases, base_mangle)
  932. grammar_path = os.path.join(*dotted_path) + EXT
  933. to_try = self.import_paths + ([base_path] if base_path is not None else []) + [stdlib_loader]
  934. for source in to_try:
  935. try:
  936. if callable(source):
  937. joined_path, text = source(base_path, grammar_path)
  938. else:
  939. joined_path = os.path.join(source, grammar_path)
  940. with open(joined_path, encoding='utf8') as f:
  941. text = f.read()
  942. except IOError:
  943. continue
  944. else:
  945. h = hashlib.md5(text.encode('utf8')).hexdigest()
  946. if self.used_files.get(joined_path, h) != h:
  947. raise RuntimeError("Grammar file was changed during importing")
  948. self.used_files[joined_path] = h
  949. gb = GrammarBuilder(self.global_keep_all_tokens, self.import_paths, self.used_files)
  950. gb.load_grammar(text, joined_path, mangle)
  951. gb._remove_unused(map(mangle, aliases))
  952. for name in gb._definitions:
  953. if name in self._definitions:
  954. raise GrammarError("Cannot import '%s' from '%s': Symbol already defined." % (name, grammar_path))
  955. self._definitions.update(**gb._definitions)
  956. break
  957. else:
  958. # Search failed. Make Python throw a nice error.
  959. open(grammar_path, encoding='utf8')
  960. assert False, "Couldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for it" % (dotted_path,)
  961. def validate(self):
  962. for name, (params, exp, _options) in self._definitions.items():
  963. for i, p in enumerate(params):
  964. if p in self._definitions:
  965. raise GrammarError("Template Parameter conflicts with rule %s (in template %s)" % (p, name))
  966. if p in params[:i]:
  967. raise GrammarError("Duplicate Template Parameter %s (in template %s)" % (p, name))
  968. if exp is None: # Remaining checks don't apply to abstract rules/terminals
  969. continue
  970. for temp in exp.find_data('template_usage'):
  971. sym = temp.children[0]
  972. args = temp.children[1:]
  973. if sym not in params:
  974. if sym not in self._definitions:
  975. self._grammar_error("Template '%s' used but not defined (in {type} {name})" % sym, name)
  976. if len(args) != len(self._definitions[sym][0]):
  977. expected, actual = len(self._definitions[sym][0]), len(args)
  978. self._grammar_error("Wrong number of template arguments used for {name} "
  979. "(expected %s, got %s) (in {type2} {name2})" % (expected, actual), sym, name)
  980. for sym in _find_used_symbols(exp):
  981. if sym not in self._definitions and sym not in params:
  982. self._grammar_error("{Type} '{name}' used but not defined (in {type2} {name2})", sym, name)
  983. if not set(self._definitions).issuperset(self._ignore_names):
  984. raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(self._ignore_names) - set(self._definitions)))
  985. def build(self):
  986. self.validate()
  987. rule_defs = []
  988. term_defs = []
  989. for name, (params, exp, options) in self._definitions.items():
  990. if self._is_term(name):
  991. assert len(params) == 0
  992. term_defs.append((name, (exp, options)))
  993. else:
  994. rule_defs.append((name, params, exp, options))
  995. # resolve_term_references(term_defs)
  996. return Grammar(rule_defs, term_defs, self._ignore_names)
  997. def verify_used_files(file_hashes):
  998. for path, old in file_hashes.items():
  999. text = None
  1000. if isinstance(path, str) and os.path.exists(path):
  1001. with open(path, encoding='utf8') as f:
  1002. text = f.read()
  1003. elif isinstance(path, PackageResource):
  1004. with suppress(IOError):
  1005. text = pkgutil.get_data(*path).decode('utf-8')
  1006. if text is None: # We don't know how to load the path. ignore it.
  1007. continue
  1008. current = hashlib.md5(text.encode()).hexdigest()
  1009. if old != current:
  1010. logger.info("File %r changed, rebuilding Parser" % path)
  1011. return False
  1012. return True
  1013. def load_grammar(grammar, source, import_paths, global_keep_all_tokens):
  1014. builder = GrammarBuilder(global_keep_all_tokens, import_paths)
  1015. builder.load_grammar(grammar, source)
  1016. return builder.build(), builder.used_files