This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1354 lines
50 KiB

  1. """Parses and creates Grammar objects"""
  2. import hashlib
  3. import os.path
  4. import sys
  5. from collections import namedtuple
  6. from copy import copy, deepcopy
  7. from io import open
  8. import pkgutil
  9. from ast import literal_eval
  10. from numbers import Integral
  11. from .utils import bfs, Py36, logger, classify_bool, is_id_continue, is_id_start, bfs_all_unique, small_factors
  12. from .lexer import Token, TerminalDef, PatternStr, PatternRE
  13. from .parse_tree_builder import ParseTreeBuilder
  14. from .parser_frontends import ParsingFrontend
  15. from .common import LexerConf, ParserConf
  16. from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
  17. from .utils import classify, suppress, dedup_list, Str
  18. from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError
  19. from .tree import Tree, SlottedTree as ST
  20. from .visitors import Transformer, Visitor, v_args, Transformer_InPlace, Transformer_NonRecursive
  21. inline_args = v_args(inline=True)
  22. __path__ = os.path.dirname(__file__)
  23. IMPORT_PATHS = ['grammars']
  24. EXT = '.lark'
  25. _RE_FLAGS = 'imslux'
  26. _EMPTY = Symbol('__empty__')
  27. _TERMINAL_NAMES = {
  28. '.' : 'DOT',
  29. ',' : 'COMMA',
  30. ':' : 'COLON',
  31. ';' : 'SEMICOLON',
  32. '+' : 'PLUS',
  33. '-' : 'MINUS',
  34. '*' : 'STAR',
  35. '/' : 'SLASH',
  36. '\\' : 'BACKSLASH',
  37. '|' : 'VBAR',
  38. '?' : 'QMARK',
  39. '!' : 'BANG',
  40. '@' : 'AT',
  41. '#' : 'HASH',
  42. '$' : 'DOLLAR',
  43. '%' : 'PERCENT',
  44. '^' : 'CIRCUMFLEX',
  45. '&' : 'AMPERSAND',
  46. '_' : 'UNDERSCORE',
  47. '<' : 'LESSTHAN',
  48. '>' : 'MORETHAN',
  49. '=' : 'EQUAL',
  50. '"' : 'DBLQUOTE',
  51. '\'' : 'QUOTE',
  52. '`' : 'BACKQUOTE',
  53. '~' : 'TILDE',
  54. '(' : 'LPAR',
  55. ')' : 'RPAR',
  56. '{' : 'LBRACE',
  57. '}' : 'RBRACE',
  58. '[' : 'LSQB',
  59. ']' : 'RSQB',
  60. '\n' : 'NEWLINE',
  61. '\r\n' : 'CRLF',
  62. '\t' : 'TAB',
  63. ' ' : 'SPACE',
  64. }
  65. # Grammar Parser
  66. TERMINALS = {
  67. '_LPAR': r'\(',
  68. '_RPAR': r'\)',
  69. '_LBRA': r'\[',
  70. '_RBRA': r'\]',
  71. '_LBRACE': r'\{',
  72. '_RBRACE': r'\}',
  73. 'OP': '[+*]|[?](?![a-z])',
  74. '_COLON': ':',
  75. '_COMMA': ',',
  76. '_OR': r'\|',
  77. '_DOT': r'\.(?!\.)',
  78. '_DOTDOT': r'\.\.',
  79. 'TILDE': '~',
  80. 'RULE': '!?[_?]?[a-z][_a-z0-9]*',
  81. 'TERMINAL': '_?[A-Z][_A-Z0-9]*',
  82. 'STRING': r'"(\\"|\\\\|[^"\n])*?"i?',
  83. 'REGEXP': r'/(?!/)(\\/|\\\\|[^/])*?/[%s]*' % _RE_FLAGS,
  84. '_NL': r'(\r?\n)+\s*',
  85. '_NL_OR': r'(\r?\n)+\s*\|',
  86. 'WS': r'[ \t]+',
  87. 'COMMENT': r'\s*//[^\n]*',
  88. '_TO': '->',
  89. '_IGNORE': r'%ignore',
  90. '_OVERRIDE': r'%override',
  91. '_DECLARE': r'%declare',
  92. '_EXTEND': r'%extend',
  93. '_IMPORT': r'%import',
  94. 'NUMBER': r'[+-]?\d+',
  95. }
  96. RULES = {
  97. 'start': ['_list'],
  98. '_list': ['_item', '_list _item'],
  99. '_item': ['rule', 'term', 'ignore', 'import', 'declare', 'override', 'extend', '_NL'],
  100. 'rule': ['RULE template_params _COLON expansions _NL',
  101. 'RULE template_params _DOT NUMBER _COLON expansions _NL'],
  102. 'template_params': ['_LBRACE _template_params _RBRACE',
  103. ''],
  104. '_template_params': ['RULE',
  105. '_template_params _COMMA RULE'],
  106. 'expansions': ['_expansions'],
  107. '_expansions': ['alias',
  108. '_expansions _OR alias',
  109. '_expansions _NL_OR alias'],
  110. '?alias': ['expansion _TO RULE', 'expansion'],
  111. 'expansion': ['_expansion'],
  112. '_expansion': ['', '_expansion expr'],
  113. '?expr': ['atom',
  114. 'atom OP',
  115. 'atom TILDE NUMBER',
  116. 'atom TILDE NUMBER _DOTDOT NUMBER',
  117. ],
  118. '?atom': ['_LPAR expansions _RPAR',
  119. 'maybe',
  120. 'value'],
  121. 'value': ['terminal',
  122. 'nonterminal',
  123. 'literal',
  124. 'range',
  125. 'template_usage'],
  126. 'terminal': ['TERMINAL'],
  127. 'nonterminal': ['RULE'],
  128. '?name': ['RULE', 'TERMINAL'],
  129. 'maybe': ['_LBRA expansions _RBRA'],
  130. 'range': ['STRING _DOTDOT STRING'],
  131. 'template_usage': ['RULE _LBRACE _template_args _RBRACE'],
  132. '_template_args': ['value',
  133. '_template_args _COMMA value'],
  134. 'term': ['TERMINAL _COLON expansions _NL',
  135. 'TERMINAL _DOT NUMBER _COLON expansions _NL'],
  136. 'override': ['_OVERRIDE rule',
  137. '_OVERRIDE term'],
  138. 'extend': ['_EXTEND rule',
  139. '_EXTEND term'],
  140. 'ignore': ['_IGNORE expansions _NL'],
  141. 'declare': ['_DECLARE _declare_args _NL'],
  142. 'import': ['_IMPORT _import_path _NL',
  143. '_IMPORT _import_path _LPAR name_list _RPAR _NL',
  144. '_IMPORT _import_path _TO name _NL'],
  145. '_import_path': ['import_lib', 'import_rel'],
  146. 'import_lib': ['_import_args'],
  147. 'import_rel': ['_DOT _import_args'],
  148. '_import_args': ['name', '_import_args _DOT name'],
  149. 'name_list': ['_name_list'],
  150. '_name_list': ['name', '_name_list _COMMA name'],
  151. '_declare_args': ['name', '_declare_args name'],
  152. 'literal': ['REGEXP', 'STRING'],
  153. }
  154. # Value 5 keeps the number of states in the lalr parser somewhat minimal
  155. # It isn't optimal, but close to it. See PR #949
  156. SMALL_FACTOR_THRESHOLD = 5
  157. # The Threshold whether repeat via ~ are split up into different rules
  158. # 50 is chosen since it keeps the number of states low and therefore lalr analysis time low,
  159. # while not being to overaggressive and unnecessarily creating rules that might create shift/reduce conflicts.
  160. # (See PR #949)
  161. REPEAT_BREAK_THRESHOLD = 50
  162. @inline_args
  163. class EBNF_to_BNF(Transformer_InPlace):
  164. def __init__(self):
  165. self.new_rules = []
  166. self.rules_cache = {}
  167. self.prefix = 'anon'
  168. self.i = 0
  169. self.rule_options = None
  170. def _name_rule(self, inner):
  171. new_name = '__%s_%s_%d' % (self.prefix, inner, self.i)
  172. self.i += 1
  173. return new_name
  174. def _add_rule(self, key, name, expansions):
  175. t = NonTerminal(name)
  176. self.new_rules.append((name, expansions, self.rule_options))
  177. self.rules_cache[key] = t
  178. return t
  179. def _add_recurse_rule(self, type_, expr):
  180. try:
  181. return self.rules_cache[expr]
  182. except KeyError:
  183. new_name = self._name_rule(type_)
  184. t = NonTerminal(new_name)
  185. tree = ST('expansions', [
  186. ST('expansion', [expr]),
  187. ST('expansion', [t, expr])
  188. ])
  189. return self._add_rule(expr, new_name, tree)
  190. def _add_repeat_rule(self, a, b, target, atom):
  191. """Generate a rule that repeats target ``a`` times, and repeats atom ``b`` times.
  192. When called recursively (into target), it repeats atom for x(n) times, where:
  193. x(0) = 1
  194. x(n) = a(n) * x(n-1) + b
  195. Example rule when a=3, b=4:
  196. new_rule: target target target atom atom atom atom
  197. """
  198. key = (a, b, target, atom)
  199. try:
  200. return self.rules_cache[key]
  201. except KeyError:
  202. new_name = self._name_rule('repeat_a%d_b%d' % (a, b))
  203. tree = ST('expansions', [ST('expansion', [target] * a + [atom] * b)])
  204. return self._add_rule(key, new_name, tree)
  205. def _add_repeat_opt_rule(self, a, b, target, target_opt, atom):
  206. """Creates a rule that matches atom 0 to (a*n+b)-1 times.
  207. When target matches n times atom, and target_opt 0 to n-1 times target_opt,
  208. First we generate target * i followed by target_opt, for i from 0 to a-1
  209. These match 0 to n*a - 1 times atom
  210. Then we generate target * a followed by atom * i, for i from 0 to b-1
  211. These match n*a to n*a + b-1 times atom
  212. The created rule will not have any shift/reduce conflicts so that it can be used with lalr
  213. Example rule when a=3, b=4:
  214. new_rule: target_opt
  215. | target target_opt
  216. | target target target_opt
  217. | target target target
  218. | target target target atom
  219. | target target target atom atom
  220. | target target target atom atom atom
  221. """
  222. key = (a, b, target, atom, "opt")
  223. try:
  224. return self.rules_cache[key]
  225. except KeyError:
  226. new_name = self._name_rule('repeat_a%d_b%d_opt' % (a, b))
  227. tree = ST('expansions', [
  228. ST('expansion', [target]*i + [target_opt]) for i in range(a)
  229. ] + [
  230. ST('expansion', [target]*a + [atom]*i) for i in range(b)
  231. ])
  232. return self._add_rule(key, new_name, tree)
  233. def _generate_repeats(self, rule, mn, mx):
  234. """Generates a rule tree that repeats ``rule`` exactly between ``mn`` to ``mx`` times.
  235. """
  236. # For a small number of repeats, we can take the naive approach
  237. if mx < REPEAT_BREAK_THRESHOLD:
  238. return ST('expansions', [ST('expansion', [rule] * n) for n in range(mn, mx + 1)])
  239. # For large repeat values, we break the repetition into sub-rules.
  240. # We treat ``rule~mn..mx`` as ``rule~mn rule~0..(diff=mx-mn)``.
  241. # We then use small_factors to split up mn and diff up into values [(a, b), ...]
  242. # This values are used with the help of _add_repeat_rule and _add_repeat_rule_opt
  243. # to generate a complete rule/expression that matches the corresponding number of repeats
  244. mn_target = rule
  245. for a, b in small_factors(mn, SMALL_FACTOR_THRESHOLD):
  246. mn_target = self._add_repeat_rule(a, b, mn_target, rule)
  247. if mx == mn:
  248. return mn_target
  249. diff = mx - mn + 1 # We add one because _add_repeat_opt_rule generates rules that match one less
  250. diff_factors = small_factors(diff, SMALL_FACTOR_THRESHOLD)
  251. diff_target = rule # Match rule 1 times
  252. diff_opt_target = ST('expansion', []) # match rule 0 times (e.g. up to 1 -1 times)
  253. for a, b in diff_factors[:-1]:
  254. diff_opt_target = self._add_repeat_opt_rule(a, b, diff_target, diff_opt_target, rule)
  255. diff_target = self._add_repeat_rule(a, b, diff_target, rule)
  256. a, b = diff_factors[-1]
  257. diff_opt_target = self._add_repeat_opt_rule(a, b, diff_target, diff_opt_target, rule)
  258. return ST('expansions', [ST('expansion', [mn_target] + [diff_opt_target])])
  259. def expr(self, rule, op, *args):
  260. if op.value == '?':
  261. empty = ST('expansion', [])
  262. return ST('expansions', [rule, empty])
  263. elif op.value == '+':
  264. # a : b c+ d
  265. # -->
  266. # a : b _c d
  267. # _c : _c c | c;
  268. return self._add_recurse_rule('plus', rule)
  269. elif op.value == '*':
  270. # a : b c* d
  271. # -->
  272. # a : b _c? d
  273. # _c : _c c | c;
  274. new_name = self._add_recurse_rule('star', rule)
  275. return ST('expansions', [new_name, ST('expansion', [])])
  276. elif op.value == '~':
  277. if len(args) == 1:
  278. mn = mx = int(args[0])
  279. else:
  280. mn, mx = map(int, args)
  281. if mx < mn or mn < 0:
  282. raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (rule, mn, mx))
  283. return self._generate_repeats(rule, mn, mx)
  284. assert False, op
  285. def maybe(self, rule):
  286. keep_all_tokens = self.rule_options and self.rule_options.keep_all_tokens
  287. def will_not_get_removed(sym):
  288. if isinstance(sym, NonTerminal):
  289. return not sym.name.startswith('_')
  290. if isinstance(sym, Terminal):
  291. return keep_all_tokens or not sym.filter_out
  292. assert False
  293. if any(rule.scan_values(will_not_get_removed)):
  294. empty = _EMPTY
  295. else:
  296. empty = ST('expansion', [])
  297. return ST('expansions', [rule, empty])
  298. class SimplifyRule_Visitor(Visitor):
  299. @staticmethod
  300. def _flatten(tree):
  301. while tree.expand_kids_by_data(tree.data):
  302. pass
  303. def expansion(self, tree):
  304. # rules_list unpacking
  305. # a : b (c|d) e
  306. # -->
  307. # a : b c e | b d e
  308. #
  309. # In AST terms:
  310. # expansion(b, expansions(c, d), e)
  311. # -->
  312. # expansions( expansion(b, c, e), expansion(b, d, e) )
  313. self._flatten(tree)
  314. for i, child in enumerate(tree.children):
  315. if isinstance(child, Tree) and child.data == 'expansions':
  316. tree.data = 'expansions'
  317. tree.children = [self.visit(ST('expansion', [option if i == j else other
  318. for j, other in enumerate(tree.children)]))
  319. for option in dedup_list(child.children)]
  320. self._flatten(tree)
  321. break
  322. def alias(self, tree):
  323. rule, alias_name = tree.children
  324. if rule.data == 'expansions':
  325. aliases = []
  326. for child in tree.children[0].children:
  327. aliases.append(ST('alias', [child, alias_name]))
  328. tree.data = 'expansions'
  329. tree.children = aliases
  330. def expansions(self, tree):
  331. self._flatten(tree)
  332. # Ensure all children are unique
  333. if len(set(tree.children)) != len(tree.children):
  334. tree.children = dedup_list(tree.children) # dedup is expensive, so try to minimize its use
  335. class RuleTreeToText(Transformer):
  336. def expansions(self, x):
  337. return x
  338. def expansion(self, symbols):
  339. return symbols, None
  340. def alias(self, x):
  341. (expansion, _alias), alias = x
  342. assert _alias is None, (alias, expansion, '-', _alias) # Double alias not allowed
  343. return expansion, alias.value
  344. class PrepareAnonTerminals(Transformer_InPlace):
  345. """Create a unique list of anonymous terminals. Attempt to give meaningful names to them when we add them"""
  346. def __init__(self, terminals):
  347. self.terminals = terminals
  348. self.term_set = {td.name for td in self.terminals}
  349. self.term_reverse = {td.pattern: td for td in terminals}
  350. self.i = 0
  351. self.rule_options = None
  352. @inline_args
  353. def pattern(self, p):
  354. value = p.value
  355. if p in self.term_reverse and p.flags != self.term_reverse[p].pattern.flags:
  356. raise GrammarError(u'Conflicting flags for the same terminal: %s' % p)
  357. term_name = None
  358. if isinstance(p, PatternStr):
  359. try:
  360. # If already defined, use the user-defined terminal name
  361. term_name = self.term_reverse[p].name
  362. except KeyError:
  363. # Try to assign an indicative anon-terminal name
  364. try:
  365. term_name = _TERMINAL_NAMES[value]
  366. except KeyError:
  367. if value and is_id_continue(value) and is_id_start(value[0]) and value.upper() not in self.term_set:
  368. term_name = value.upper()
  369. if term_name in self.term_set:
  370. term_name = None
  371. elif isinstance(p, PatternRE):
  372. if p in self.term_reverse: # Kind of a weird placement.name
  373. term_name = self.term_reverse[p].name
  374. else:
  375. assert False, p
  376. if term_name is None:
  377. term_name = '__ANON_%d' % self.i
  378. self.i += 1
  379. if term_name not in self.term_set:
  380. assert p not in self.term_reverse
  381. self.term_set.add(term_name)
  382. termdef = TerminalDef(term_name, p)
  383. self.term_reverse[p] = termdef
  384. self.terminals.append(termdef)
  385. filter_out = False if self.rule_options and self.rule_options.keep_all_tokens else isinstance(p, PatternStr)
  386. return Terminal(term_name, filter_out=filter_out)
  387. class _ReplaceSymbols(Transformer_InPlace):
  388. """Helper for ApplyTemplates"""
  389. def __init__(self):
  390. self.names = {}
  391. def value(self, c):
  392. if len(c) == 1 and isinstance(c[0], Token) and c[0].value in self.names:
  393. return self.names[c[0].value]
  394. return self.__default__('value', c, None)
  395. def template_usage(self, c):
  396. if c[0] in self.names:
  397. return self.__default__('template_usage', [self.names[c[0]].name] + c[1:], None)
  398. return self.__default__('template_usage', c, None)
  399. class ApplyTemplates(Transformer_InPlace):
  400. """Apply the templates, creating new rules that represent the used templates"""
  401. def __init__(self, rule_defs):
  402. self.rule_defs = rule_defs
  403. self.replacer = _ReplaceSymbols()
  404. self.created_templates = set()
  405. def template_usage(self, c):
  406. name = c[0]
  407. args = c[1:]
  408. result_name = "%s{%s}" % (name, ",".join(a.name for a in args))
  409. if result_name not in self.created_templates:
  410. self.created_templates.add(result_name)
  411. (_n, params, tree, options) ,= (t for t in self.rule_defs if t[0] == name)
  412. assert len(params) == len(args), args
  413. result_tree = deepcopy(tree)
  414. self.replacer.names = dict(zip(params, args))
  415. self.replacer.transform(result_tree)
  416. self.rule_defs.append((result_name, [], result_tree, deepcopy(options)))
  417. return NonTerminal(result_name)
  418. def _rfind(s, choices):
  419. return max(s.rfind(c) for c in choices)
  420. def eval_escaping(s):
  421. w = ''
  422. i = iter(s)
  423. for n in i:
  424. w += n
  425. if n == '\\':
  426. try:
  427. n2 = next(i)
  428. except StopIteration:
  429. raise GrammarError("Literal ended unexpectedly (bad escaping): `%r`" % s)
  430. if n2 == '\\':
  431. w += '\\\\'
  432. elif n2 not in 'Uuxnftr':
  433. w += '\\'
  434. w += n2
  435. w = w.replace('\\"', '"').replace("'", "\\'")
  436. to_eval = "u'''%s'''" % w
  437. try:
  438. s = literal_eval(to_eval)
  439. except SyntaxError as e:
  440. raise GrammarError(s, e)
  441. return s
  442. def _literal_to_pattern(literal):
  443. v = literal.value
  444. flag_start = _rfind(v, '/"')+1
  445. assert flag_start > 0
  446. flags = v[flag_start:]
  447. assert all(f in _RE_FLAGS for f in flags), flags
  448. if literal.type == 'STRING' and '\n' in v:
  449. raise GrammarError('You cannot put newlines in string literals')
  450. if literal.type == 'REGEXP' and '\n' in v and 'x' not in flags:
  451. raise GrammarError('You can only use newlines in regular expressions '
  452. 'with the `x` (verbose) flag')
  453. v = v[:flag_start]
  454. assert v[0] == v[-1] and v[0] in '"/'
  455. x = v[1:-1]
  456. s = eval_escaping(x)
  457. if s == "":
  458. raise GrammarError("Empty terminals are not allowed (%s)" % literal)
  459. if literal.type == 'STRING':
  460. s = s.replace('\\\\', '\\')
  461. return PatternStr(s, flags, raw=literal.value)
  462. elif literal.type == 'REGEXP':
  463. return PatternRE(s, flags, raw=literal.value)
  464. else:
  465. assert False, 'Invariant failed: literal.type not in ["STRING", "REGEXP"]'
  466. @inline_args
  467. class PrepareLiterals(Transformer_InPlace):
  468. def literal(self, literal):
  469. return ST('pattern', [_literal_to_pattern(literal)])
  470. def range(self, start, end):
  471. assert start.type == end.type == 'STRING'
  472. start = start.value[1:-1]
  473. end = end.value[1:-1]
  474. assert len(eval_escaping(start)) == len(eval_escaping(end)) == 1
  475. regexp = '[%s-%s]' % (start, end)
  476. return ST('pattern', [PatternRE(regexp)])
  477. def _make_joined_pattern(regexp, flags_set):
  478. # In Python 3.6, a new syntax for flags was introduced, that allows us to restrict the scope
  479. # of flags to a specific regexp group. We are already using it in `lexer.Pattern._get_flags`
  480. # However, for prior Python versions, we still need to use global flags, so we have to make sure
  481. # that there are no flag collisions when we merge several terminals.
  482. flags = ()
  483. if not Py36:
  484. if len(flags_set) > 1:
  485. raise GrammarError("Lark doesn't support joining terminals with conflicting flags in python <3.6!")
  486. elif len(flags_set) == 1:
  487. flags ,= flags_set
  488. return PatternRE(regexp, flags)
  489. class TerminalTreeToPattern(Transformer_NonRecursive):
  490. def pattern(self, ps):
  491. p ,= ps
  492. return p
  493. def expansion(self, items):
  494. assert items
  495. if len(items) == 1:
  496. return items[0]
  497. pattern = ''.join(i.to_regexp() for i in items)
  498. return _make_joined_pattern(pattern, {i.flags for i in items})
  499. def expansions(self, exps):
  500. if len(exps) == 1:
  501. return exps[0]
  502. # Do a bit of sorting to make sure that the longest option is returned
  503. # (Python's re module otherwise prefers just 'l' when given (l|ll) and both could match)
  504. exps.sort(key=lambda x: (-x.max_width, -x.min_width, -len(x.value)))
  505. pattern = '(?:%s)' % ('|'.join(i.to_regexp() for i in exps))
  506. return _make_joined_pattern(pattern, {i.flags for i in exps})
  507. def expr(self, args):
  508. inner, op = args[:2]
  509. if op == '~':
  510. if len(args) == 3:
  511. op = "{%d}" % int(args[2])
  512. else:
  513. mn, mx = map(int, args[2:])
  514. if mx < mn:
  515. raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (inner, mn, mx))
  516. op = "{%d,%d}" % (mn, mx)
  517. else:
  518. assert len(args) == 2
  519. return PatternRE('(?:%s)%s' % (inner.to_regexp(), op), inner.flags)
  520. def maybe(self, expr):
  521. return self.expr(expr + ['?'])
  522. def alias(self, t):
  523. raise GrammarError("Aliasing not allowed in terminals (You used -> in the wrong place)")
  524. def value(self, v):
  525. return v[0]
  526. class PrepareSymbols(Transformer_InPlace):
  527. def value(self, v):
  528. v ,= v
  529. if isinstance(v, Tree):
  530. return v
  531. elif v.type == 'RULE':
  532. return NonTerminal(Str(v.value))
  533. elif v.type == 'TERMINAL':
  534. return Terminal(Str(v.value), filter_out=v.startswith('_'))
  535. assert False
  536. def nr_deepcopy_tree(t):
  537. """Deepcopy tree `t` without recursion"""
  538. return Transformer_NonRecursive(False).transform(t)
  539. class Grammar:
  540. def __init__(self, rule_defs, term_defs, ignore):
  541. self.term_defs = term_defs
  542. self.rule_defs = rule_defs
  543. self.ignore = ignore
  544. def compile(self, start, terminals_to_keep):
  545. # We change the trees in-place (to support huge grammars)
  546. # So deepcopy allows calling compile more than once.
  547. term_defs = [(n, (nr_deepcopy_tree(t), p)) for n, (t, p) in self.term_defs]
  548. rule_defs = [(n, p, nr_deepcopy_tree(t), o) for n, p, t, o in self.rule_defs]
  549. # ===================
  550. # Compile Terminals
  551. # ===================
  552. # Convert terminal-trees to strings/regexps
  553. for name, (term_tree, priority) in term_defs:
  554. if term_tree is None: # Terminal added through %declare
  555. continue
  556. expansions = list(term_tree.find_data('expansion'))
  557. if len(expansions) == 1 and not expansions[0].children:
  558. raise GrammarError("Terminals cannot be empty (%s)" % name)
  559. transformer = PrepareLiterals() * TerminalTreeToPattern()
  560. terminals = [TerminalDef(name, transformer.transform(term_tree), priority)
  561. for name, (term_tree, priority) in term_defs if term_tree]
  562. # =================
  563. # Compile Rules
  564. # =================
  565. # 1. Pre-process terminals
  566. anon_tokens_transf = PrepareAnonTerminals(terminals)
  567. transformer = PrepareLiterals() * PrepareSymbols() * anon_tokens_transf # Adds to terminals
  568. # 2. Inline Templates
  569. transformer *= ApplyTemplates(rule_defs)
  570. # 3. Convert EBNF to BNF (and apply step 1 & 2)
  571. ebnf_to_bnf = EBNF_to_BNF()
  572. rules = []
  573. i = 0
  574. while i < len(rule_defs): # We have to do it like this because rule_defs might grow due to templates
  575. name, params, rule_tree, options = rule_defs[i]
  576. i += 1
  577. if len(params) != 0: # Dont transform templates
  578. continue
  579. rule_options = RuleOptions(keep_all_tokens=True) if options and options.keep_all_tokens else None
  580. ebnf_to_bnf.rule_options = rule_options
  581. ebnf_to_bnf.prefix = name
  582. anon_tokens_transf.rule_options = rule_options
  583. tree = transformer.transform(rule_tree)
  584. res = ebnf_to_bnf.transform(tree)
  585. rules.append((name, res, options))
  586. rules += ebnf_to_bnf.new_rules
  587. assert len(rules) == len({name for name, _t, _o in rules}), "Whoops, name collision"
  588. # 4. Compile tree to Rule objects
  589. rule_tree_to_text = RuleTreeToText()
  590. simplify_rule = SimplifyRule_Visitor()
  591. compiled_rules = []
  592. for rule_content in rules:
  593. name, tree, options = rule_content
  594. simplify_rule.visit(tree)
  595. expansions = rule_tree_to_text.transform(tree)
  596. for i, (expansion, alias) in enumerate(expansions):
  597. if alias and name.startswith('_'):
  598. raise GrammarError("Rule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)"% (name, alias))
  599. empty_indices = [x==_EMPTY for x in expansion]
  600. if any(empty_indices):
  601. exp_options = copy(options) or RuleOptions()
  602. exp_options.empty_indices = empty_indices
  603. expansion = [x for x in expansion if x!=_EMPTY]
  604. else:
  605. exp_options = options
  606. for sym in expansion:
  607. assert isinstance(sym, Symbol)
  608. if sym.is_term and exp_options and exp_options.keep_all_tokens:
  609. sym.filter_out = False
  610. rule = Rule(NonTerminal(name), expansion, i, alias, exp_options)
  611. compiled_rules.append(rule)
  612. # Remove duplicates of empty rules, throw error for non-empty duplicates
  613. if len(set(compiled_rules)) != len(compiled_rules):
  614. duplicates = classify(compiled_rules, lambda x: x)
  615. for dups in duplicates.values():
  616. if len(dups) > 1:
  617. if dups[0].expansion:
  618. raise GrammarError("Rules defined twice: %s\n\n(Might happen due to colliding expansion of optionals: [] or ?)"
  619. % ''.join('\n * %s' % i for i in dups))
  620. # Empty rule; assert all other attributes are equal
  621. assert len({(r.alias, r.order, r.options) for r in dups}) == len(dups)
  622. # Remove duplicates
  623. compiled_rules = list(set(compiled_rules))
  624. # Filter out unused rules
  625. while True:
  626. c = len(compiled_rules)
  627. used_rules = {s for r in compiled_rules
  628. for s in r.expansion
  629. if isinstance(s, NonTerminal)
  630. and s != r.origin}
  631. used_rules |= {NonTerminal(s) for s in start}
  632. compiled_rules, unused = classify_bool(compiled_rules, lambda r: r.origin in used_rules)
  633. for r in unused:
  634. logger.debug("Unused rule: %s", r)
  635. if len(compiled_rules) == c:
  636. break
  637. # Filter out unused terminals
  638. if terminals_to_keep != '*':
  639. used_terms = {t.name for r in compiled_rules
  640. for t in r.expansion
  641. if isinstance(t, Terminal)}
  642. terminals, unused = classify_bool(terminals, lambda t: t.name in used_terms or t.name in self.ignore or t.name in terminals_to_keep)
  643. if unused:
  644. logger.debug("Unused terminals: %s", [t.name for t in unused])
  645. return terminals, compiled_rules, self.ignore
  646. PackageResource = namedtuple('PackageResource', 'pkg_name path')
  647. class FromPackageLoader(object):
  648. """
  649. Provides a simple way of creating custom import loaders that load from packages via ``pkgutil.get_data`` instead of using `open`.
  650. This allows them to be compatible even from within zip files.
  651. Relative imports are handled, so you can just freely use them.
  652. pkg_name: The name of the package. You can probably provide `__name__` most of the time
  653. search_paths: All the path that will be search on absolute imports.
  654. """
  655. def __init__(self, pkg_name, search_paths=("", )):
  656. self.pkg_name = pkg_name
  657. self.search_paths = search_paths
  658. def __repr__(self):
  659. return "%s(%r, %r)" % (type(self).__name__, self.pkg_name, self.search_paths)
  660. def __call__(self, base_path, grammar_path):
  661. if base_path is None:
  662. to_try = self.search_paths
  663. else:
  664. # Check whether or not the importing grammar was loaded by this module.
  665. if not isinstance(base_path, PackageResource) or base_path.pkg_name != self.pkg_name:
  666. # Technically false, but FileNotFound doesn't exist in python2.7, and this message should never reach the end user anyway
  667. raise IOError()
  668. to_try = [base_path.path]
  669. for path in to_try:
  670. full_path = os.path.join(path, grammar_path)
  671. try:
  672. text = pkgutil.get_data(self.pkg_name, full_path)
  673. except IOError:
  674. continue
  675. else:
  676. return PackageResource(self.pkg_name, full_path), text.decode()
  677. raise IOError()
  678. stdlib_loader = FromPackageLoader('lark', IMPORT_PATHS)
  679. def resolve_term_references(term_dict):
  680. # TODO Solve with transitive closure (maybe)
  681. while True:
  682. changed = False
  683. for name, token_tree in term_dict.items():
  684. if token_tree is None: # Terminal added through %declare
  685. continue
  686. for exp in token_tree.find_data('value'):
  687. item ,= exp.children
  688. if isinstance(item, Token):
  689. if item.type == 'RULE':
  690. raise GrammarError("Rules aren't allowed inside terminals (%s in %s)" % (item, name))
  691. if item.type == 'TERMINAL':
  692. try:
  693. term_value = term_dict[item]
  694. except KeyError:
  695. raise GrammarError("Terminal used but not defined: %s" % item)
  696. assert term_value is not None
  697. exp.children[0] = term_value
  698. changed = True
  699. if not changed:
  700. break
  701. for name, term in term_dict.items():
  702. if term: # Not just declared
  703. for child in term.children:
  704. ids = [id(x) for x in child.iter_subtrees()]
  705. if id(term) in ids:
  706. raise GrammarError("Recursion in terminal '%s' (recursion is only allowed in rules, not terminals)" % name)
  707. def options_from_rule(name, params, *x):
  708. if len(x) > 1:
  709. priority, expansions = x
  710. priority = int(priority)
  711. else:
  712. expansions ,= x
  713. priority = None
  714. params = [t.value for t in params.children] if params is not None else [] # For the grammar parser
  715. keep_all_tokens = name.startswith('!')
  716. name = name.lstrip('!')
  717. expand1 = name.startswith('?')
  718. name = name.lstrip('?')
  719. return name, params, expansions, RuleOptions(keep_all_tokens, expand1, priority=priority,
  720. template_source=(name if params else None))
  721. def symbols_from_strcase(expansion):
  722. return [Terminal(x, filter_out=x.startswith('_')) if x.isupper() else NonTerminal(x) for x in expansion]
  723. @inline_args
  724. class PrepareGrammar(Transformer_InPlace):
  725. def terminal(self, name):
  726. return name
  727. def nonterminal(self, name):
  728. return name
  729. def _find_used_symbols(tree):
  730. assert tree.data == 'expansions'
  731. return {t for x in tree.find_data('expansion')
  732. for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
  733. def _get_parser():
  734. try:
  735. return _get_parser.cache
  736. except AttributeError:
  737. terminals = [TerminalDef(name, PatternRE(value)) for name, value in TERMINALS.items()]
  738. rules = [options_from_rule(name, None, x) for name, x in RULES.items()]
  739. rules = [Rule(NonTerminal(r), symbols_from_strcase(x.split()), i, None, o)
  740. for r, _p, xs, o in rules for i, x in enumerate(xs)]
  741. callback = ParseTreeBuilder(rules, ST).create_callback()
  742. import re
  743. lexer_conf = LexerConf(terminals, re, ['WS', 'COMMENT'])
  744. parser_conf = ParserConf(rules, callback, ['start'])
  745. lexer_conf.lexer_type = 'standard'
  746. parser_conf.parser_type = 'lalr'
  747. _get_parser.cache = ParsingFrontend(lexer_conf, parser_conf, None)
  748. return _get_parser.cache
  749. GRAMMAR_ERRORS = [
  750. ('Incorrect type of value', ['a: 1\n']),
  751. ('Unclosed parenthesis', ['a: (\n']),
  752. ('Unmatched closing parenthesis', ['a: )\n', 'a: [)\n', 'a: (]\n']),
  753. ('Expecting rule or terminal definition (missing colon)', ['a\n', 'A\n', 'a->\n', 'A->\n', 'a A\n']),
  754. ('Illegal name for rules or terminals', ['Aa:\n']),
  755. ('Alias expects lowercase name', ['a: -> "a"\n']),
  756. ('Unexpected colon', ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n']),
  757. ('Misplaced operator', ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n']),
  758. ('Expecting option ("|") or a new rule or terminal definition', ['a:a\n()\n']),
  759. ('Terminal names cannot contain dots', ['A.B\n']),
  760. ('Expecting rule or terminal definition', ['"a"\n']),
  761. ('%import expects a name', ['%import "a"\n']),
  762. ('%ignore expects a value', ['%ignore %import\n']),
  763. ]
  764. def _translate_parser_exception(parse, e):
  765. error = e.match_examples(parse, GRAMMAR_ERRORS, use_accepts=True)
  766. if error:
  767. return error
  768. elif 'STRING' in e.expected:
  769. return "Expecting a value"
  770. def _parse_grammar(text, name, start='start'):
  771. try:
  772. tree = _get_parser().parse(text + '\n', start)
  773. except UnexpectedCharacters as e:
  774. context = e.get_context(text)
  775. raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
  776. (e.line, e.column, name, context))
  777. except UnexpectedToken as e:
  778. context = e.get_context(text)
  779. error = _translate_parser_exception(_get_parser().parse, e)
  780. if error:
  781. raise GrammarError("%s, at line %s column %s\n\n%s" % (error, e.line, e.column, context))
  782. raise
  783. return PrepareGrammar().transform(tree)
  784. def _error_repr(error):
  785. if isinstance(error, UnexpectedToken):
  786. error2 = _translate_parser_exception(_get_parser().parse, error)
  787. if error2:
  788. return error2
  789. expected = ', '.join(error.accepts or error.expected)
  790. return "Unexpected token %r. Expected one of: {%s}" % (str(error.token), expected)
  791. else:
  792. return str(error)
  793. def _search_interactive_parser(interactive_parser, predicate):
  794. def expand(node):
  795. path, p = node
  796. for choice in p.choices():
  797. t = Token(choice, '')
  798. try:
  799. new_p = p.feed_token(t)
  800. except ParseError: # Illegal
  801. pass
  802. else:
  803. yield path + (choice,), new_p
  804. for path, p in bfs_all_unique([((), interactive_parser)], expand):
  805. if predicate(p):
  806. return path, p
  807. def find_grammar_errors(text, start='start'):
  808. errors = []
  809. def on_error(e):
  810. errors.append((e, _error_repr(e)))
  811. # recover to a new line
  812. token_path, _ = _search_interactive_parser(e.interactive_parser.as_immutable(), lambda p: '_NL' in p.choices())
  813. for token_type in token_path:
  814. e.interactive_parser.feed_token(Token(token_type, ''))
  815. e.interactive_parser.feed_token(Token('_NL', '\n'))
  816. return True
  817. _tree = _get_parser().parse(text + '\n', start, on_error=on_error)
  818. errors_by_line = classify(errors, lambda e: e[0].line)
  819. errors = [el[0] for el in errors_by_line.values()] # already sorted
  820. for e in errors:
  821. e[0].interactive_parser = None
  822. return errors
  823. def _get_mangle(prefix, aliases, base_mangle=None):
  824. def mangle(s):
  825. if s in aliases:
  826. s = aliases[s]
  827. else:
  828. if s[0] == '_':
  829. s = '_%s__%s' % (prefix, s[1:])
  830. else:
  831. s = '%s__%s' % (prefix, s)
  832. if base_mangle is not None:
  833. s = base_mangle(s)
  834. return s
  835. return mangle
  836. def _mangle_exp(exp, mangle):
  837. if mangle is None:
  838. return exp
  839. exp = deepcopy(exp) # TODO: is this needed
  840. for t in exp.iter_subtrees():
  841. for i, c in enumerate(t.children):
  842. if isinstance(c, Token) and c.type in ('RULE', 'TERMINAL'):
  843. t.children[i] = Token(c.type, mangle(c.value))
  844. return exp
  845. class GrammarBuilder:
  846. def __init__(self, global_keep_all_tokens=False, import_paths=None, used_files=None):
  847. self.global_keep_all_tokens = global_keep_all_tokens
  848. self.import_paths = import_paths or []
  849. self.used_files = used_files or {}
  850. self._definitions = {}
  851. self._ignore_names = []
  852. def _is_term(self, name):
  853. # Imported terminals are of the form `Path__to__Grammar__file__TERMINAL_NAME`
  854. # Only the last part is the actual name, and the rest might contain mixed case
  855. return name.rpartition('__')[-1].isupper()
  856. def _grammar_error(self, msg, *names):
  857. args = {}
  858. for i, name in enumerate(names, start=1):
  859. postfix = '' if i == 1 else str(i)
  860. args['name' + postfix] = name
  861. args['type' + postfix] = lowercase_type = ("rule", "terminal")[self._is_term(name)]
  862. args['Type' + postfix] = lowercase_type.title()
  863. raise GrammarError(msg.format(**args))
  864. def _check_options(self, name, options):
  865. if self._is_term(name):
  866. if options is None:
  867. options = 1
  868. # if we don't use Integral here, we run into python2.7/python3 problems with long vs int
  869. elif not isinstance(options, Integral):
  870. raise GrammarError("Terminal require a single int as 'options' (e.g. priority), got %s" % (type(options),))
  871. else:
  872. if options is None:
  873. options = RuleOptions()
  874. elif not isinstance(options, RuleOptions):
  875. raise GrammarError("Rules require a RuleOptions instance as 'options'")
  876. if self.global_keep_all_tokens:
  877. options.keep_all_tokens = True
  878. return options
  879. def _define(self, name, exp, params=(), options=None, override=False):
  880. if name in self._definitions:
  881. if not override:
  882. self._grammar_error("{Type} '{name}' defined more than once", name)
  883. elif override:
  884. self._grammar_error("Cannot override a nonexisting {type} {name}", name)
  885. if name.startswith('__'):
  886. self._grammar_error('Names starting with double-underscore are reserved (Error at {name})', name)
  887. self._definitions[name] = (params, exp, self._check_options(name, options))
  888. def _extend(self, name, exp, params=(), options=None):
  889. if name not in self._definitions:
  890. self._grammar_error("Can't extend {type} {name} as it wasn't defined before", name)
  891. if tuple(params) != tuple(self._definitions[name][0]):
  892. self._grammar_error("Cannot extend {type} with different parameters: {name}", name)
  893. # TODO: think about what to do with 'options'
  894. base = self._definitions[name][1]
  895. assert isinstance(base, Tree) and base.data == 'expansions'
  896. base.children.insert(0, exp)
  897. def _ignore(self, exp_or_name):
  898. if isinstance(exp_or_name, str):
  899. self._ignore_names.append(exp_or_name)
  900. else:
  901. assert isinstance(exp_or_name, Tree)
  902. t = exp_or_name
  903. if t.data == 'expansions' and len(t.children) == 1:
  904. t2 ,= t.children
  905. if t2.data=='expansion' and len(t2.children) == 1:
  906. item ,= t2.children
  907. if item.data == 'value':
  908. item ,= item.children
  909. if isinstance(item, Token) and item.type == 'TERMINAL':
  910. self._ignore_names.append(item.value)
  911. return
  912. name = '__IGNORE_%d'% len(self._ignore_names)
  913. self._ignore_names.append(name)
  914. self._definitions[name] = ((), t, 1)
  915. def _declare(self, *names):
  916. for name in names:
  917. self._define(name, None)
  918. def _unpack_import(self, stmt, grammar_name):
  919. if len(stmt.children) > 1:
  920. path_node, arg1 = stmt.children
  921. else:
  922. path_node, = stmt.children
  923. arg1 = None
  924. if isinstance(arg1, Tree): # Multi import
  925. dotted_path = tuple(path_node.children)
  926. names = arg1.children
  927. aliases = dict(zip(names, names)) # Can't have aliased multi import, so all aliases will be the same as names
  928. else: # Single import
  929. dotted_path = tuple(path_node.children[:-1])
  930. if not dotted_path:
  931. name ,= path_node.children
  932. raise GrammarError("Nothing was imported from grammar `%s`" % name)
  933. name = path_node.children[-1] # Get name from dotted path
  934. aliases = {name.value: (arg1 or name).value} # Aliases if exist
  935. if path_node.data == 'import_lib': # Import from library
  936. base_path = None
  937. else: # Relative import
  938. if grammar_name == '<string>': # Import relative to script file path if grammar is coded in script
  939. try:
  940. base_file = os.path.abspath(sys.modules['__main__'].__file__)
  941. except AttributeError:
  942. base_file = None
  943. else:
  944. base_file = grammar_name # Import relative to grammar file path if external grammar file
  945. if base_file:
  946. if isinstance(base_file, PackageResource):
  947. base_path = PackageResource(base_file.pkg_name, os.path.split(base_file.path)[0])
  948. else:
  949. base_path = os.path.split(base_file)[0]
  950. else:
  951. base_path = os.path.abspath(os.path.curdir)
  952. return dotted_path, base_path, aliases
  953. def _unpack_definition(self, tree, mangle):
  954. if tree.data == 'rule':
  955. name, params, exp, opts = options_from_rule(*tree.children)
  956. else:
  957. name = tree.children[0].value
  958. params = () # TODO terminal templates
  959. opts = int(tree.children[1]) if len(tree.children) == 3 else 1 # priority
  960. exp = tree.children[-1]
  961. if mangle is not None:
  962. params = tuple(mangle(p) for p in params)
  963. name = mangle(name)
  964. exp = _mangle_exp(exp, mangle)
  965. return name, exp, params, opts
  966. def load_grammar(self, grammar_text, grammar_name="<?>", mangle=None):
  967. tree = _parse_grammar(grammar_text, grammar_name)
  968. imports = {}
  969. for stmt in tree.children:
  970. if stmt.data == 'import':
  971. dotted_path, base_path, aliases = self._unpack_import(stmt, grammar_name)
  972. try:
  973. import_base_path, import_aliases = imports[dotted_path]
  974. assert base_path == import_base_path, 'Inconsistent base_path for %s.' % '.'.join(dotted_path)
  975. import_aliases.update(aliases)
  976. except KeyError:
  977. imports[dotted_path] = base_path, aliases
  978. for dotted_path, (base_path, aliases) in imports.items():
  979. self.do_import(dotted_path, base_path, aliases, mangle)
  980. for stmt in tree.children:
  981. if stmt.data in ('term', 'rule'):
  982. self._define(*self._unpack_definition(stmt, mangle))
  983. elif stmt.data == 'override':
  984. r ,= stmt.children
  985. self._define(*self._unpack_definition(r, mangle), override=True)
  986. elif stmt.data == 'extend':
  987. r ,= stmt.children
  988. self._extend(*self._unpack_definition(r, mangle))
  989. elif stmt.data == 'ignore':
  990. # if mangle is not None, we shouldn't apply ignore, since we aren't in a toplevel grammar
  991. if mangle is None:
  992. self._ignore(*stmt.children)
  993. elif stmt.data == 'declare':
  994. names = [t.value for t in stmt.children]
  995. if mangle is None:
  996. self._declare(*names)
  997. else:
  998. self._declare(*map(mangle, names))
  999. elif stmt.data == 'import':
  1000. pass
  1001. else:
  1002. assert False, stmt
  1003. term_defs = { name: exp
  1004. for name, (_params, exp, _options) in self._definitions.items()
  1005. if self._is_term(name)
  1006. }
  1007. resolve_term_references(term_defs)
  1008. def _remove_unused(self, used):
  1009. def rule_dependencies(symbol):
  1010. if self._is_term(symbol):
  1011. return []
  1012. try:
  1013. params, tree,_ = self._definitions[symbol]
  1014. except KeyError:
  1015. return []
  1016. return _find_used_symbols(tree) - set(params)
  1017. _used = set(bfs(used, rule_dependencies))
  1018. self._definitions = {k: v for k, v in self._definitions.items() if k in _used}
  1019. def do_import(self, dotted_path, base_path, aliases, base_mangle=None):
  1020. assert dotted_path
  1021. mangle = _get_mangle('__'.join(dotted_path), aliases, base_mangle)
  1022. grammar_path = os.path.join(*dotted_path) + EXT
  1023. to_try = self.import_paths + ([base_path] if base_path is not None else []) + [stdlib_loader]
  1024. for source in to_try:
  1025. try:
  1026. if callable(source):
  1027. joined_path, text = source(base_path, grammar_path)
  1028. else:
  1029. joined_path = os.path.join(source, grammar_path)
  1030. with open(joined_path, encoding='utf8') as f:
  1031. text = f.read()
  1032. except IOError:
  1033. continue
  1034. else:
  1035. h = hashlib.md5(text.encode('utf8')).hexdigest()
  1036. if self.used_files.get(joined_path, h) != h:
  1037. raise RuntimeError("Grammar file was changed during importing")
  1038. self.used_files[joined_path] = h
  1039. gb = GrammarBuilder(self.global_keep_all_tokens, self.import_paths, self.used_files)
  1040. gb.load_grammar(text, joined_path, mangle)
  1041. gb._remove_unused(map(mangle, aliases))
  1042. for name in gb._definitions:
  1043. if name in self._definitions:
  1044. raise GrammarError("Cannot import '%s' from '%s': Symbol already defined." % (name, grammar_path))
  1045. self._definitions.update(**gb._definitions)
  1046. break
  1047. else:
  1048. # Search failed. Make Python throw a nice error.
  1049. open(grammar_path, encoding='utf8')
  1050. assert False, "Couldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for it" % (dotted_path,)
  1051. def validate(self):
  1052. for name, (params, exp, _options) in self._definitions.items():
  1053. for i, p in enumerate(params):
  1054. if p in self._definitions:
  1055. raise GrammarError("Template Parameter conflicts with rule %s (in template %s)" % (p, name))
  1056. if p in params[:i]:
  1057. raise GrammarError("Duplicate Template Parameter %s (in template %s)" % (p, name))
  1058. if exp is None: # Remaining checks don't apply to abstract rules/terminals
  1059. continue
  1060. for temp in exp.find_data('template_usage'):
  1061. sym = temp.children[0]
  1062. args = temp.children[1:]
  1063. if sym not in params:
  1064. if sym not in self._definitions:
  1065. self._grammar_error("Template '%s' used but not defined (in {type} {name})" % sym, name)
  1066. if len(args) != len(self._definitions[sym][0]):
  1067. expected, actual = len(self._definitions[sym][0]), len(args)
  1068. self._grammar_error("Wrong number of template arguments used for {name} "
  1069. "(expected %s, got %s) (in {type2} {name2})" % (expected, actual), sym, name)
  1070. for sym in _find_used_symbols(exp):
  1071. if sym not in self._definitions and sym not in params:
  1072. self._grammar_error("{Type} '{name}' used but not defined (in {type2} {name2})", sym, name)
  1073. if not set(self._definitions).issuperset(self._ignore_names):
  1074. raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(self._ignore_names) - set(self._definitions)))
  1075. def build(self):
  1076. self.validate()
  1077. rule_defs = []
  1078. term_defs = []
  1079. for name, (params, exp, options) in self._definitions.items():
  1080. if self._is_term(name):
  1081. assert len(params) == 0
  1082. term_defs.append((name, (exp, options)))
  1083. else:
  1084. rule_defs.append((name, params, exp, options))
  1085. # resolve_term_references(term_defs)
  1086. return Grammar(rule_defs, term_defs, self._ignore_names)
  1087. def verify_used_files(file_hashes):
  1088. for path, old in file_hashes.items():
  1089. text = None
  1090. if isinstance(path, str) and os.path.exists(path):
  1091. with open(path, encoding='utf8') as f:
  1092. text = f.read()
  1093. elif isinstance(path, PackageResource):
  1094. with suppress(IOError):
  1095. text = pkgutil.get_data(*path).decode('utf-8')
  1096. if text is None: # We don't know how to load the path. ignore it.
  1097. continue
  1098. current = hashlib.md5(text.encode()).hexdigest()
  1099. if old != current:
  1100. logger.info("File %r changed, rebuilding Parser" % path)
  1101. return False
  1102. return True
  1103. def list_grammar_imports(grammar, import_paths=[]):
  1104. "Returns a list of paths to the lark grammars imported by the given grammar (recursively)"
  1105. builder = GrammarBuilder(False, import_paths)
  1106. builder.load_grammar(grammar, '<string>')
  1107. return list(builder.used_files.keys())
  1108. def load_grammar(grammar, source, import_paths, global_keep_all_tokens):
  1109. builder = GrammarBuilder(global_keep_all_tokens, import_paths)
  1110. builder.load_grammar(grammar, source)
  1111. return builder.build(), builder.used_files