This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1225 lines
44 KiB

  1. """Parses and creates Grammar objects"""
  2. import os.path
  3. import sys
  4. from copy import copy, deepcopy
  5. from io import open
  6. import pkgutil
  7. from ast import literal_eval
  8. from numbers import Integral
  9. from .utils import bfs, Py36, logger, classify_bool, is_id_continue, is_id_start, bfs_all_unique
  10. from .lexer import Token, TerminalDef, PatternStr, PatternRE
  11. from .parse_tree_builder import ParseTreeBuilder
  12. from .parser_frontends import ParsingFrontend
  13. from .common import LexerConf, ParserConf
  14. from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol, END
  15. from .utils import classify, suppress, dedup_list, Str
  16. from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError
  17. from .tree import Tree, SlottedTree as ST
  18. from .visitors import Transformer, Visitor, v_args, Transformer_InPlace, Transformer_NonRecursive
  19. inline_args = v_args(inline=True)
  20. __path__ = os.path.dirname(__file__)
  21. IMPORT_PATHS = ['grammars']
  22. EXT = '.lark'
  23. _RE_FLAGS = 'imslux'
  24. _EMPTY = Symbol('__empty__')
  25. _TERMINAL_NAMES = {
  26. '.' : 'DOT',
  27. ',' : 'COMMA',
  28. ':' : 'COLON',
  29. ';' : 'SEMICOLON',
  30. '+' : 'PLUS',
  31. '-' : 'MINUS',
  32. '*' : 'STAR',
  33. '/' : 'SLASH',
  34. '\\' : 'BACKSLASH',
  35. '|' : 'VBAR',
  36. '?' : 'QMARK',
  37. '!' : 'BANG',
  38. '@' : 'AT',
  39. '#' : 'HASH',
  40. '$' : 'DOLLAR',
  41. '%' : 'PERCENT',
  42. '^' : 'CIRCUMFLEX',
  43. '&' : 'AMPERSAND',
  44. '_' : 'UNDERSCORE',
  45. '<' : 'LESSTHAN',
  46. '>' : 'MORETHAN',
  47. '=' : 'EQUAL',
  48. '"' : 'DBLQUOTE',
  49. '\'' : 'QUOTE',
  50. '`' : 'BACKQUOTE',
  51. '~' : 'TILDE',
  52. '(' : 'LPAR',
  53. ')' : 'RPAR',
  54. '{' : 'LBRACE',
  55. '}' : 'RBRACE',
  56. '[' : 'LSQB',
  57. ']' : 'RSQB',
  58. '\n' : 'NEWLINE',
  59. '\r\n' : 'CRLF',
  60. '\t' : 'TAB',
  61. ' ' : 'SPACE',
  62. }
  63. # Grammar Parser
  64. TERMINALS = {
  65. '_LPAR': r'\(',
  66. '_RPAR': r'\)',
  67. '_LBRA': r'\[',
  68. '_RBRA': r'\]',
  69. '_LBRACE': r'\{',
  70. '_RBRACE': r'\}',
  71. 'OP': '[+*]|[?](?![a-z])',
  72. '_COLON': ':',
  73. '_COMMA': ',',
  74. '_OR': r'\|',
  75. '_DOT': r'\.(?!\.)',
  76. '_DOTDOT': r'\.\.',
  77. 'TILDE': '~',
  78. 'RULE': '!?[_?]?[a-z][_a-z0-9]*',
  79. 'TERMINAL': '_?[A-Z][_A-Z0-9]*',
  80. 'STRING': r'"(\\"|\\\\|[^"\n])*?"i?',
  81. 'REGEXP': r'/(?!/)(\\/|\\\\|[^/])*?/[%s]*' % _RE_FLAGS,
  82. '_NL': r'(\r?\n)+\s*',
  83. 'WS': r'[ \t]+',
  84. 'COMMENT': r'\s*//[^\n]*',
  85. '_TO': '->',
  86. '_IGNORE': r'%ignore',
  87. '_OVERRIDE': r'%override',
  88. '_DECLARE': r'%declare',
  89. '_EXTEND': r'%extend',
  90. '_IMPORT': r'%import',
  91. 'NUMBER': r'[+-]?\d+',
  92. '_END': r'\$',
  93. }
  94. RULES = {
  95. 'start': ['_list'],
  96. '_list': ['_item', '_list _item'],
  97. '_item': ['rule', 'term', 'ignore', 'import', 'declare', 'override', 'extend', '_NL'],
  98. 'rule': ['RULE template_params _COLON expansions _NL',
  99. 'RULE template_params _DOT NUMBER _COLON expansions _NL'],
  100. 'template_params': ['_LBRACE _template_params _RBRACE',
  101. ''],
  102. '_template_params': ['RULE',
  103. '_template_params _COMMA RULE'],
  104. 'expansions': ['alias',
  105. 'expansions _OR alias',
  106. 'expansions _NL _OR alias'],
  107. '?alias': ['expansion _TO RULE', 'expansion'],
  108. 'expansion': ['_expansion'],
  109. '_expansion': ['', '_expansion expr'],
  110. '?expr': ['atom',
  111. 'atom OP',
  112. 'atom TILDE NUMBER',
  113. 'atom TILDE NUMBER _DOTDOT NUMBER',
  114. ],
  115. '?atom': ['_LPAR expansions _RPAR',
  116. 'maybe',
  117. 'value'],
  118. 'value': ['terminal',
  119. 'nonterminal',
  120. 'literal',
  121. 'range',
  122. 'end',
  123. 'template_usage'],
  124. 'terminal': ['TERMINAL'],
  125. 'nonterminal': ['RULE'],
  126. '?name': ['RULE', 'TERMINAL'],
  127. 'maybe': ['_LBRA expansions _RBRA'],
  128. 'range': ['STRING _DOTDOT STRING'],
  129. 'end': ['_END'],
  130. 'template_usage': ['RULE _LBRACE _template_args _RBRACE'],
  131. '_template_args': ['value',
  132. '_template_args _COMMA value'],
  133. 'term': ['TERMINAL _COLON expansions _NL',
  134. 'TERMINAL _DOT NUMBER _COLON expansions _NL'],
  135. 'override': ['_OVERRIDE rule',
  136. '_OVERRIDE term'],
  137. 'extend': ['_EXTEND rule',
  138. '_EXTEND term'],
  139. 'ignore': ['_IGNORE expansions _NL'],
  140. 'declare': ['_DECLARE _declare_args _NL'],
  141. 'import': ['_IMPORT _import_path _NL',
  142. '_IMPORT _import_path _LPAR name_list _RPAR _NL',
  143. '_IMPORT _import_path _TO name _NL'],
  144. '_import_path': ['import_lib', 'import_rel'],
  145. 'import_lib': ['_import_args'],
  146. 'import_rel': ['_DOT _import_args'],
  147. '_import_args': ['name', '_import_args _DOT name'],
  148. 'name_list': ['_name_list'],
  149. '_name_list': ['name', '_name_list _COMMA name'],
  150. '_declare_args': ['name', '_declare_args name'],
  151. 'literal': ['REGEXP', 'STRING'],
  152. }
  153. @inline_args
  154. class EBNF_to_BNF(Transformer_InPlace):
  155. def __init__(self):
  156. self.new_rules = []
  157. self.rules_by_expr = {}
  158. self.prefix = 'anon'
  159. self.i = 0
  160. self.rule_options = None
  161. def _add_recurse_rule(self, type_, expr):
  162. if expr in self.rules_by_expr:
  163. return self.rules_by_expr[expr]
  164. new_name = '__%s_%s_%d' % (self.prefix, type_, self.i)
  165. self.i += 1
  166. t = NonTerminal(new_name)
  167. tree = ST('expansions', [ST('expansion', [expr]), ST('expansion', [t, expr])])
  168. self.new_rules.append((new_name, tree, self.rule_options))
  169. self.rules_by_expr[expr] = t
  170. return t
  171. def expr(self, rule, op, *args):
  172. if op.value == '?':
  173. empty = ST('expansion', [])
  174. return ST('expansions', [rule, empty])
  175. elif op.value == '+':
  176. # a : b c+ d
  177. # -->
  178. # a : b _c d
  179. # _c : _c c | c;
  180. return self._add_recurse_rule('plus', rule)
  181. elif op.value == '*':
  182. # a : b c* d
  183. # -->
  184. # a : b _c? d
  185. # _c : _c c | c;
  186. new_name = self._add_recurse_rule('star', rule)
  187. return ST('expansions', [new_name, ST('expansion', [])])
  188. elif op.value == '~':
  189. if len(args) == 1:
  190. mn = mx = int(args[0])
  191. else:
  192. mn, mx = map(int, args)
  193. if mx < mn or mn < 0:
  194. raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (rule, mn, mx))
  195. return ST('expansions', [ST('expansion', [rule] * n) for n in range(mn, mx+1)])
  196. assert False, op
  197. def maybe(self, rule):
  198. keep_all_tokens = self.rule_options and self.rule_options.keep_all_tokens
  199. def will_not_get_removed(sym):
  200. if isinstance(sym, NonTerminal):
  201. return not sym.name.startswith('_')
  202. if isinstance(sym, Terminal):
  203. return keep_all_tokens or not sym.filter_out
  204. assert False
  205. if any(rule.scan_values(will_not_get_removed)):
  206. empty = _EMPTY
  207. else:
  208. empty = ST('expansion', [])
  209. return ST('expansions', [rule, empty])
  210. class SimplifyRule_Visitor(Visitor):
  211. @staticmethod
  212. def _flatten(tree):
  213. while True:
  214. to_expand = [i for i, child in enumerate(tree.children)
  215. if isinstance(child, Tree) and child.data == tree.data]
  216. if not to_expand:
  217. break
  218. tree.expand_kids_by_index(*to_expand)
  219. def expansion(self, tree):
  220. # rules_list unpacking
  221. # a : b (c|d) e
  222. # -->
  223. # a : b c e | b d e
  224. #
  225. # In AST terms:
  226. # expansion(b, expansions(c, d), e)
  227. # -->
  228. # expansions( expansion(b, c, e), expansion(b, d, e) )
  229. self._flatten(tree)
  230. for i, child in enumerate(tree.children):
  231. if isinstance(child, Tree) and child.data == 'expansions':
  232. tree.data = 'expansions'
  233. tree.children = [self.visit(ST('expansion', [option if i == j else other
  234. for j, other in enumerate(tree.children)]))
  235. for option in dedup_list(child.children)]
  236. self._flatten(tree)
  237. break
  238. def alias(self, tree):
  239. rule, alias_name = tree.children
  240. if rule.data == 'expansions':
  241. aliases = []
  242. for child in tree.children[0].children:
  243. aliases.append(ST('alias', [child, alias_name]))
  244. tree.data = 'expansions'
  245. tree.children = aliases
  246. def expansions(self, tree):
  247. self._flatten(tree)
  248. # Ensure all children are unique
  249. if len(set(tree.children)) != len(tree.children):
  250. tree.children = dedup_list(tree.children) # dedup is expensive, so try to minimize its use
  251. class RuleTreeToText(Transformer):
  252. def expansions(self, x):
  253. return x
  254. def expansion(self, symbols):
  255. return symbols, None
  256. def alias(self, x):
  257. (expansion, _alias), alias = x
  258. assert _alias is None, (alias, expansion, '-', _alias) # Double alias not allowed
  259. return expansion, alias.value
  260. class PrepareAnonTerminals(Transformer_InPlace):
  261. """Create a unique list of anonymous terminals. Attempt to give meaningful names to them when we add them"""
  262. def __init__(self, terminals):
  263. self.terminals = terminals
  264. self.term_set = {td.name for td in self.terminals}
  265. self.term_reverse = {td.pattern: td for td in terminals}
  266. self.i = 0
  267. self.rule_options = None
  268. @inline_args
  269. def pattern(self, p):
  270. value = p.value
  271. if p in self.term_reverse and p.flags != self.term_reverse[p].pattern.flags:
  272. raise GrammarError(u'Conflicting flags for the same terminal: %s' % p)
  273. term_name = None
  274. if isinstance(p, PatternStr):
  275. try:
  276. # If already defined, use the user-defined terminal name
  277. term_name = self.term_reverse[p].name
  278. except KeyError:
  279. # Try to assign an indicative anon-terminal name
  280. try:
  281. term_name = _TERMINAL_NAMES[value]
  282. except KeyError:
  283. if is_id_continue(value) and is_id_start(value[0]) and value.upper() not in self.term_set:
  284. term_name = value.upper()
  285. if term_name in self.term_set:
  286. term_name = None
  287. elif isinstance(p, PatternRE):
  288. if p in self.term_reverse: # Kind of a weird placement.name
  289. term_name = self.term_reverse[p].name
  290. else:
  291. assert False, p
  292. if term_name is None:
  293. term_name = '__ANON_%d' % self.i
  294. self.i += 1
  295. if term_name not in self.term_set:
  296. assert p not in self.term_reverse
  297. self.term_set.add(term_name)
  298. termdef = TerminalDef(term_name, p)
  299. self.term_reverse[p] = termdef
  300. self.terminals.append(termdef)
  301. filter_out = False if self.rule_options and self.rule_options.keep_all_tokens else isinstance(p, PatternStr)
  302. return Terminal(term_name, filter_out=filter_out)
  303. class _ReplaceSymbols(Transformer_InPlace):
  304. """Helper for ApplyTemplates"""
  305. def __init__(self):
  306. self.names = {}
  307. def value(self, c):
  308. if len(c) == 1 and isinstance(c[0], Token) and c[0].value in self.names:
  309. return self.names[c[0].value]
  310. return self.__default__('value', c, None)
  311. def template_usage(self, c):
  312. if c[0] in self.names:
  313. return self.__default__('template_usage', [self.names[c[0]].name] + c[1:], None)
  314. return self.__default__('template_usage', c, None)
  315. class ApplyTemplates(Transformer_InPlace):
  316. """Apply the templates, creating new rules that represent the used templates"""
  317. def __init__(self, rule_defs):
  318. self.rule_defs = rule_defs
  319. self.replacer = _ReplaceSymbols()
  320. self.created_templates = set()
  321. def template_usage(self, c):
  322. name = c[0]
  323. args = c[1:]
  324. result_name = "%s{%s}" % (name, ",".join(a.name for a in args))
  325. if result_name not in self.created_templates:
  326. self.created_templates.add(result_name)
  327. (_n, params, tree, options) ,= (t for t in self.rule_defs if t[0] == name)
  328. assert len(params) == len(args), args
  329. result_tree = deepcopy(tree)
  330. self.replacer.names = dict(zip(params, args))
  331. self.replacer.transform(result_tree)
  332. self.rule_defs.append((result_name, [], result_tree, deepcopy(options)))
  333. return NonTerminal(result_name)
  334. def _rfind(s, choices):
  335. return max(s.rfind(c) for c in choices)
  336. def eval_escaping(s):
  337. w = ''
  338. i = iter(s)
  339. for n in i:
  340. w += n
  341. if n == '\\':
  342. try:
  343. n2 = next(i)
  344. except StopIteration:
  345. raise GrammarError("Literal ended unexpectedly (bad escaping): `%r`" % s)
  346. if n2 == '\\':
  347. w += '\\\\'
  348. elif n2 not in 'uxnftr':
  349. w += '\\'
  350. w += n2
  351. w = w.replace('\\"', '"').replace("'", "\\'")
  352. to_eval = "u'''%s'''" % w
  353. try:
  354. s = literal_eval(to_eval)
  355. except SyntaxError as e:
  356. raise GrammarError(s, e)
  357. return s
  358. def _literal_to_pattern(literal):
  359. v = literal.value
  360. flag_start = _rfind(v, '/"')+1
  361. assert flag_start > 0
  362. flags = v[flag_start:]
  363. assert all(f in _RE_FLAGS for f in flags), flags
  364. if literal.type == 'STRING' and '\n' in v:
  365. raise GrammarError('You cannot put newlines in string literals')
  366. if literal.type == 'REGEXP' and '\n' in v and 'x' not in flags:
  367. raise GrammarError('You can only use newlines in regular expressions '
  368. 'with the `x` (verbose) flag')
  369. v = v[:flag_start]
  370. assert v[0] == v[-1] and v[0] in '"/'
  371. x = v[1:-1]
  372. s = eval_escaping(x)
  373. if literal.type == 'STRING':
  374. s = s.replace('\\\\', '\\')
  375. return PatternStr(s, flags, raw=literal.value)
  376. elif literal.type == 'REGEXP':
  377. return PatternRE(s, flags, raw=literal.value)
  378. else:
  379. assert False, 'Invariant failed: literal.type not in ["STRING", "REGEXP"]'
  380. @inline_args
  381. class PrepareLiterals(Transformer_InPlace):
  382. def literal(self, literal):
  383. return ST('pattern', [_literal_to_pattern(literal)])
  384. def range(self, start, end):
  385. assert start.type == end.type == 'STRING'
  386. start = start.value[1:-1]
  387. end = end.value[1:-1]
  388. assert len(eval_escaping(start)) == len(eval_escaping(end)) == 1
  389. regexp = '[%s-%s]' % (start, end)
  390. return ST('pattern', [PatternRE(regexp)])
  391. def _make_joined_pattern(regexp, flags_set):
  392. # In Python 3.6, a new syntax for flags was introduced, that allows us to restrict the scope
  393. # of flags to a specific regexp group. We are already using it in `lexer.Pattern._get_flags`
  394. # However, for prior Python versions, we still need to use global flags, so we have to make sure
  395. # that there are no flag collisions when we merge several terminals.
  396. flags = ()
  397. if not Py36:
  398. if len(flags_set) > 1:
  399. raise GrammarError("Lark doesn't support joining terminals with conflicting flags in python <3.6!")
  400. elif len(flags_set) == 1:
  401. flags ,= flags_set
  402. return PatternRE(regexp, flags)
  403. class TerminalTreeToPattern(Transformer):
  404. def pattern(self, ps):
  405. p ,= ps
  406. return p
  407. def expansion(self, items):
  408. assert items
  409. if len(items) == 1:
  410. return items[0]
  411. pattern = ''.join(i.to_regexp() for i in items)
  412. return _make_joined_pattern(pattern, {i.flags for i in items})
  413. def expansions(self, exps):
  414. if len(exps) == 1:
  415. return exps[0]
  416. pattern = '(?:%s)' % ('|'.join(i.to_regexp() for i in exps))
  417. return _make_joined_pattern(pattern, {i.flags for i in exps})
  418. def expr(self, args):
  419. inner, op = args[:2]
  420. if op == '~':
  421. if len(args) == 3:
  422. op = "{%d}" % int(args[2])
  423. else:
  424. mn, mx = map(int, args[2:])
  425. if mx < mn:
  426. raise GrammarError("Bad Range for %s (%d..%d isn't allowed)" % (inner, mn, mx))
  427. op = "{%d,%d}" % (mn, mx)
  428. else:
  429. assert len(args) == 2
  430. return PatternRE('(?:%s)%s' % (inner.to_regexp(), op), inner.flags)
  431. def maybe(self, expr):
  432. return self.expr(expr + ['?'])
  433. def alias(self, t):
  434. raise GrammarError("Aliasing not allowed in terminals (You used -> in the wrong place)")
  435. def value(self, v):
  436. return v[0]
  437. class PrepareSymbols(Transformer_InPlace):
  438. def value(self, v):
  439. v ,= v
  440. if isinstance(v, Tree):
  441. return v
  442. elif v.type == 'RULE':
  443. return NonTerminal(Str(v.value))
  444. elif v.type == 'TERMINAL':
  445. return Terminal(Str(v.value), filter_out=v.startswith('_'))
  446. assert False
  447. def nr_deepcopy_tree(t):
  448. """Deepcopy tree `t` without recursion"""
  449. return Transformer_NonRecursive(False).transform(t)
  450. class Grammar:
  451. def __init__(self, rule_defs, term_defs, ignore):
  452. self.term_defs = term_defs
  453. self.rule_defs = rule_defs
  454. self.ignore = ignore
  455. def compile(self, start, terminals_to_keep):
  456. # We change the trees in-place (to support huge grammars)
  457. # So deepcopy allows calling compile more than once.
  458. term_defs = deepcopy(list(self.term_defs))
  459. rule_defs = [(n,p,nr_deepcopy_tree(t),o) for n,p,t,o in self.rule_defs]
  460. # ===================
  461. # Compile Terminals
  462. # ===================
  463. # Convert terminal-trees to strings/regexps
  464. for name, (term_tree, priority) in term_defs:
  465. if term_tree is None: # Terminal added through %declare
  466. continue
  467. expansions = list(term_tree.find_data('expansion'))
  468. if len(expansions) == 1 and not expansions[0].children:
  469. raise GrammarError("Terminals cannot be empty (%s)" % name)
  470. transformer = PrepareLiterals() * TerminalTreeToPattern()
  471. terminals = [TerminalDef(name, transformer.transform(term_tree), priority)
  472. for name, (term_tree, priority) in term_defs if term_tree]
  473. # =================
  474. # Compile Rules
  475. # =================
  476. # 1. Pre-process terminals
  477. anon_tokens_transf = PrepareAnonTerminals(terminals)
  478. transformer = PrepareLiterals() * PrepareSymbols() * anon_tokens_transf # Adds to terminals
  479. # 2. Inline Templates
  480. transformer *= ApplyTemplates(rule_defs)
  481. # 3. Convert EBNF to BNF (and apply step 1 & 2)
  482. ebnf_to_bnf = EBNF_to_BNF()
  483. rules = []
  484. i = 0
  485. while i < len(rule_defs): # We have to do it like this because rule_defs might grow due to templates
  486. name, params, rule_tree, options = rule_defs[i]
  487. i += 1
  488. if len(params) != 0: # Dont transform templates
  489. continue
  490. rule_options = RuleOptions(keep_all_tokens=True) if options and options.keep_all_tokens else None
  491. ebnf_to_bnf.rule_options = rule_options
  492. ebnf_to_bnf.prefix = name
  493. anon_tokens_transf.rule_options = rule_options
  494. tree = transformer.transform(rule_tree)
  495. res = ebnf_to_bnf.transform(tree)
  496. rules.append((name, res, options))
  497. rules += ebnf_to_bnf.new_rules
  498. assert len(rules) == len({name for name, _t, _o in rules}), "Whoops, name collision"
  499. # 4. Compile tree to Rule objects
  500. rule_tree_to_text = RuleTreeToText()
  501. simplify_rule = SimplifyRule_Visitor()
  502. compiled_rules = []
  503. for rule_content in rules:
  504. name, tree, options = rule_content
  505. simplify_rule.visit(tree)
  506. expansions = rule_tree_to_text.transform(tree)
  507. for i, (expansion, alias) in enumerate(expansions):
  508. if alias and name.startswith('_'):
  509. raise GrammarError("Rule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)"% (name, alias))
  510. empty_indices = [x==_EMPTY for x in expansion]
  511. if any(empty_indices):
  512. exp_options = copy(options) or RuleOptions()
  513. exp_options.empty_indices = empty_indices
  514. expansion = [x for x in expansion if x!=_EMPTY]
  515. else:
  516. exp_options = options
  517. assert all(isinstance(x, Symbol) for x in expansion), expansion
  518. rule = Rule(NonTerminal(name), expansion, i, alias, exp_options)
  519. compiled_rules.append(rule)
  520. # Remove duplicates of empty rules, throw error for non-empty duplicates
  521. if len(set(compiled_rules)) != len(compiled_rules):
  522. duplicates = classify(compiled_rules, lambda x: x)
  523. for dups in duplicates.values():
  524. if len(dups) > 1:
  525. if dups[0].expansion:
  526. raise GrammarError("Rules defined twice: %s\n\n(Might happen due to colliding expansion of optionals: [] or ?)"
  527. % ''.join('\n * %s' % i for i in dups))
  528. # Empty rule; assert all other attributes are equal
  529. assert len({(r.alias, r.order, r.options) for r in dups}) == len(dups)
  530. # Remove duplicates
  531. compiled_rules = list(set(compiled_rules))
  532. # Filter out unused rules
  533. while True:
  534. c = len(compiled_rules)
  535. used_rules = {s for r in compiled_rules
  536. for s in r.expansion
  537. if isinstance(s, NonTerminal)
  538. and s != r.origin}
  539. used_rules |= {NonTerminal(s) for s in start}
  540. compiled_rules, unused = classify_bool(compiled_rules, lambda r: r.origin in used_rules)
  541. for r in unused:
  542. logger.debug("Unused rule: %s", r)
  543. if len(compiled_rules) == c:
  544. break
  545. # Filter out unused terminals
  546. used_terms = {t.name for r in compiled_rules
  547. for t in r.expansion
  548. if isinstance(t, Terminal)}
  549. terminals, unused = classify_bool(terminals, lambda t: t.name in used_terms or t.name in self.ignore or t.name in terminals_to_keep)
  550. if unused:
  551. logger.debug("Unused terminals: %s", [t.name for t in unused])
  552. return terminals, compiled_rules, self.ignore
  553. class PackageResource(object):
  554. """
  555. Represents a path inside a Package. Used by `FromPackageLoader`
  556. """
  557. def __init__(self, pkg_name, path):
  558. self.pkg_name = pkg_name
  559. self.path = path
  560. def __str__(self):
  561. return "<%s: %s>" % (self.pkg_name, self.path)
  562. def __repr__(self):
  563. return "%s(%r, %r)" % (type(self).__name__, self.pkg_name, self.path)
  564. class FromPackageLoader(object):
  565. """
  566. Provides a simple way of creating custom import loaders that load from packages via ``pkgutil.get_data`` instead of using `open`.
  567. This allows them to be compatible even from within zip files.
  568. Relative imports are handled, so you can just freely use them.
  569. pkg_name: The name of the package. You can probably provide `__name__` most of the time
  570. search_paths: All the path that will be search on absolute imports.
  571. """
  572. def __init__(self, pkg_name, search_paths=("", )):
  573. self.pkg_name = pkg_name
  574. self.search_paths = search_paths
  575. def __repr__(self):
  576. return "%s(%r, %r)" % (type(self).__name__, self.pkg_name, self.search_paths)
  577. def __call__(self, base_path, grammar_path):
  578. if base_path is None:
  579. to_try = self.search_paths
  580. else:
  581. # Check whether or not the importing grammar was loaded by this module.
  582. if not isinstance(base_path, PackageResource) or base_path.pkg_name != self.pkg_name:
  583. # Technically false, but FileNotFound doesn't exist in python2.7, and this message should never reach the end user anyway
  584. raise IOError()
  585. to_try = [base_path.path]
  586. for path in to_try:
  587. full_path = os.path.join(path, grammar_path)
  588. try:
  589. text = pkgutil.get_data(self.pkg_name, full_path)
  590. except IOError:
  591. continue
  592. else:
  593. return PackageResource(self.pkg_name, full_path), text.decode()
  594. raise IOError()
  595. stdlib_loader = FromPackageLoader('lark', IMPORT_PATHS)
  596. def resolve_term_references(term_dict):
  597. # TODO Solve with transitive closure (maybe)
  598. while True:
  599. changed = False
  600. for name, token_tree in term_dict.items():
  601. if token_tree is None: # Terminal added through %declare
  602. continue
  603. for exp in token_tree.find_data('value'):
  604. item ,= exp.children
  605. if isinstance(item, Token):
  606. if item.type == 'RULE':
  607. raise GrammarError("Rules aren't allowed inside terminals (%s in %s)" % (item, name))
  608. if item.type == 'TERMINAL':
  609. try:
  610. term_value = term_dict[item]
  611. except KeyError:
  612. raise GrammarError("Terminal used but not defined: %s" % item)
  613. assert term_value is not None
  614. exp.children[0] = term_value
  615. changed = True
  616. if not changed:
  617. break
  618. for name, term in term_dict.items():
  619. if term: # Not just declared
  620. for child in term.children:
  621. ids = [id(x) for x in child.iter_subtrees()]
  622. if id(term) in ids:
  623. raise GrammarError("Recursion in terminal '%s' (recursion is only allowed in rules, not terminals)" % name)
  624. def options_from_rule(name, params, *x):
  625. if len(x) > 1:
  626. priority, expansions = x
  627. priority = int(priority)
  628. else:
  629. expansions ,= x
  630. priority = None
  631. params = [t.value for t in params.children] if params is not None else [] # For the grammar parser
  632. keep_all_tokens = name.startswith('!')
  633. name = name.lstrip('!')
  634. expand1 = name.startswith('?')
  635. name = name.lstrip('?')
  636. return name, params, expansions, RuleOptions(keep_all_tokens, expand1, priority=priority,
  637. template_source=(name if params else None))
  638. def symbols_from_strcase(expansion):
  639. return [Terminal(x, filter_out=x.startswith('_')) if x.isupper() else NonTerminal(x) for x in expansion]
  640. @inline_args
  641. class PrepareGrammar(Transformer_InPlace):
  642. def terminal(self, name):
  643. return name
  644. def nonterminal(self, name):
  645. return name
  646. def end(self):
  647. return Token('TERMINAL', END)
  648. def _find_used_symbols(tree):
  649. assert tree.data == 'expansions'
  650. return {t for x in tree.find_data('expansion')
  651. for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
  652. def _get_parser():
  653. try:
  654. return _get_parser.cache
  655. except AttributeError:
  656. terminals = [TerminalDef(name, PatternRE(value)) for name, value in TERMINALS.items()]
  657. rules = [options_from_rule(name, None, x) for name, x in RULES.items()]
  658. rules = [Rule(NonTerminal(r), symbols_from_strcase(x.split()), i, None, o)
  659. for r, _p, xs, o in rules for i, x in enumerate(xs)]
  660. callback = ParseTreeBuilder(rules, ST).create_callback()
  661. import re
  662. lexer_conf = LexerConf(terminals, re, ['WS', 'COMMENT'])
  663. parser_conf = ParserConf(rules, callback, ['start'])
  664. lexer_conf.lexer_type = 'standard'
  665. parser_conf.parser_type = 'lalr'
  666. _get_parser.cache = ParsingFrontend(lexer_conf, parser_conf, {})
  667. return _get_parser.cache
  668. GRAMMAR_ERRORS = [
  669. ('Incorrect type of value', ['a: 1\n']),
  670. ('Unclosed parenthesis', ['a: (\n']),
  671. ('Unmatched closing parenthesis', ['a: )\n', 'a: [)\n', 'a: (]\n']),
  672. ('Expecting rule or terminal definition (missing colon)', ['a\n', 'A\n', 'a->\n', 'A->\n', 'a A\n']),
  673. ('Illegal name for rules or terminals', ['Aa:\n']),
  674. ('Alias expects lowercase name', ['a: -> "a"\n']),
  675. ('Unexpected colon', ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n']),
  676. ('Misplaced operator', ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n']),
  677. ('Expecting option ("|") or a new rule or terminal definition', ['a:a\n()\n']),
  678. ('Terminal names cannot contain dots', ['A.B\n']),
  679. ('Expecting rule or terminal definition', ['"a"\n']),
  680. ('%import expects a name', ['%import "a"\n']),
  681. ('%ignore expects a value', ['%ignore %import\n']),
  682. ]
  683. def _translate_parser_exception(parse, e):
  684. error = e.match_examples(parse, GRAMMAR_ERRORS, use_accepts=True)
  685. if error:
  686. return error
  687. elif 'STRING' in e.expected:
  688. return "Expecting a value"
  689. def _parse_grammar(text, name, start='start'):
  690. try:
  691. tree = _get_parser().parse(text + '\n', start)
  692. except UnexpectedCharacters as e:
  693. context = e.get_context(text)
  694. raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
  695. (e.line, e.column, name, context))
  696. except UnexpectedToken as e:
  697. context = e.get_context(text)
  698. error = _translate_parser_exception(_get_parser().parse, e)
  699. if error:
  700. raise GrammarError("%s, at line %s column %s\n\n%s" % (error, e.line, e.column, context))
  701. raise
  702. return PrepareGrammar().transform(tree)
  703. def _error_repr(error):
  704. if isinstance(error, UnexpectedToken):
  705. error2 = _translate_parser_exception(_get_parser().parse, error)
  706. if error2:
  707. return error2
  708. expected = ', '.join(error.accepts or error.expected)
  709. return "Unexpected token %r. Expected one of: {%s}" % (str(error.token), expected)
  710. else:
  711. return str(error)
  712. def _search_interactive_parser(interactive_parser, predicate):
  713. def expand(node):
  714. path, p = node
  715. for choice in p.choices():
  716. t = Token(choice, '')
  717. try:
  718. new_p = p.feed_token(t)
  719. except ParseError: # Illegal
  720. pass
  721. else:
  722. yield path + (choice,), new_p
  723. for path, p in bfs_all_unique([((), interactive_parser)], expand):
  724. if predicate(p):
  725. return path, p
  726. def find_grammar_errors(text, start='start'):
  727. errors = []
  728. def on_error(e):
  729. errors.append((e, _error_repr(e)))
  730. # recover to a new line
  731. token_path, _ = _search_interactive_parser(e.interactive_parser.as_immutable(), lambda p: '_NL' in p.choices())
  732. for token_type in token_path:
  733. e.interactive_parser.feed_token(Token(token_type, ''))
  734. e.interactive_parser.feed_token(Token('_NL', '\n'))
  735. return True
  736. _tree = _get_parser().parse(text + '\n', start, on_error=on_error)
  737. errors_by_line = classify(errors, lambda e: e[0].line)
  738. errors = [el[0] for el in errors_by_line.values()] # already sorted
  739. for e in errors:
  740. e[0].interactive_parser = None
  741. return errors
  742. def _get_mangle(prefix, aliases, base_mangle=None):
  743. def mangle(s):
  744. if s in aliases:
  745. s = aliases[s]
  746. else:
  747. if s[0] == '_':
  748. s = '_%s__%s' % (prefix, s[1:])
  749. else:
  750. s = '%s__%s' % (prefix, s)
  751. if base_mangle is not None:
  752. s = base_mangle(s)
  753. return s
  754. return mangle
  755. def _mangle_exp(exp, mangle):
  756. if mangle is None:
  757. return exp
  758. exp = deepcopy(exp) # TODO: is this needed
  759. for t in exp.iter_subtrees():
  760. for i, c in enumerate(t.children):
  761. if isinstance(c, Token) and c.type in ('RULE', 'TERMINAL'):
  762. t.children[i] = Token(c.type, mangle(c.value))
  763. return exp
  764. class GrammarBuilder:
  765. def __init__(self, global_keep_all_tokens=False, import_paths=None):
  766. self.global_keep_all_tokens = global_keep_all_tokens
  767. self.import_paths = import_paths or []
  768. self._definitions = {}
  769. self._ignore_names = []
  770. self._definitions[END] = ((), Tree('expansions', []), self._check_options(END, None))
  771. def _is_term(self, name):
  772. # Imported terminals are of the form `Path__to__Grammar__file__TERMINAL_NAME`
  773. # Only the last part is the actual name, and the rest might contain mixed case
  774. return name.rpartition('__')[-1].isupper()
  775. def _grammar_error(self, msg, *names):
  776. args = {}
  777. for i, name in enumerate(names, start=1):
  778. postfix = '' if i == 1 else str(i)
  779. args['name' + postfix] = name
  780. args['type' + postfix] = lowercase_type = ("rule", "terminal")[self._is_term(name)]
  781. args['Type' + postfix] = lowercase_type.title()
  782. raise GrammarError(msg.format(**args))
  783. def _check_options(self, name, options):
  784. if self._is_term(name):
  785. if options is None:
  786. options = 1
  787. # if we don't use Integral here, we run into python2.7/python3 problems with long vs int
  788. elif not isinstance(options, Integral):
  789. raise GrammarError("Terminal require a single int as 'options' (e.g. priority), got %s" % (type(options),))
  790. else:
  791. if options is None:
  792. options = RuleOptions()
  793. elif not isinstance(options, RuleOptions):
  794. raise GrammarError("Rules require a RuleOptions instance as 'options'")
  795. if self.global_keep_all_tokens:
  796. options.keep_all_tokens = True
  797. return options
  798. def _define(self, name, exp, params=(), options=None, override=False):
  799. if name in self._definitions:
  800. if not override:
  801. self._grammar_error("{Type} '{name}' defined more than once", name)
  802. elif override:
  803. self._grammar_error("Cannot override a nonexisting {type} {name}", name)
  804. if name.startswith('__'):
  805. self._grammar_error('Names starting with double-underscore are reserved (Error at {name})', name)
  806. self._definitions[name] = (params, exp, self._check_options(name, options))
  807. def _extend(self, name, exp, params=(), options=None):
  808. if name not in self._definitions:
  809. self._grammar_error("Can't extend {type} {name} as it wasn't defined before", name)
  810. if tuple(params) != tuple(self._definitions[name][0]):
  811. self._grammar_error("Cannot extend {type} with different parameters: {name}", name)
  812. # TODO: think about what to do with 'options'
  813. base = self._definitions[name][1]
  814. while len(base.children) == 2:
  815. assert isinstance(base.children[0], Tree) and base.children[0].data == 'expansions', base
  816. base = base.children[0]
  817. base.children.insert(0, exp)
  818. def _ignore(self, exp_or_name):
  819. if isinstance(exp_or_name, str):
  820. self._ignore_names.append(exp_or_name)
  821. else:
  822. assert isinstance(exp_or_name, Tree)
  823. t = exp_or_name
  824. if t.data == 'expansions' and len(t.children) == 1:
  825. t2 ,= t.children
  826. if t2.data=='expansion' and len(t2.children) == 1:
  827. item ,= t2.children
  828. if item.data == 'value':
  829. item ,= item.children
  830. if isinstance(item, Token) and item.type == 'TERMINAL':
  831. self._ignore_names.append(item.value)
  832. return
  833. name = '__IGNORE_%d'% len(self._ignore_names)
  834. self._ignore_names.append(name)
  835. self._definitions[name] = ((), t, 1)
  836. def _declare(self, *names):
  837. for name in names:
  838. self._define(name, None)
  839. def _unpack_import(self, stmt, grammar_name):
  840. if len(stmt.children) > 1:
  841. path_node, arg1 = stmt.children
  842. else:
  843. path_node, = stmt.children
  844. arg1 = None
  845. if isinstance(arg1, Tree): # Multi import
  846. dotted_path = tuple(path_node.children)
  847. names = arg1.children
  848. aliases = dict(zip(names, names)) # Can't have aliased multi import, so all aliases will be the same as names
  849. else: # Single import
  850. dotted_path = tuple(path_node.children[:-1])
  851. if not dotted_path:
  852. name ,= path_node.children
  853. raise GrammarError("Nothing was imported from grammar `%s`" % name)
  854. name = path_node.children[-1] # Get name from dotted path
  855. aliases = {name.value: (arg1 or name).value} # Aliases if exist
  856. if path_node.data == 'import_lib': # Import from library
  857. base_path = None
  858. else: # Relative import
  859. if grammar_name == '<string>': # Import relative to script file path if grammar is coded in script
  860. try:
  861. base_file = os.path.abspath(sys.modules['__main__'].__file__)
  862. except AttributeError:
  863. base_file = None
  864. else:
  865. base_file = grammar_name # Import relative to grammar file path if external grammar file
  866. if base_file:
  867. if isinstance(base_file, PackageResource):
  868. base_path = PackageResource(base_file.pkg_name, os.path.split(base_file.path)[0])
  869. else:
  870. base_path = os.path.split(base_file)[0]
  871. else:
  872. base_path = os.path.abspath(os.path.curdir)
  873. return dotted_path, base_path, aliases
  874. def _unpack_definition(self, tree, mangle):
  875. if tree.data == 'rule':
  876. name, params, exp, opts = options_from_rule(*tree.children)
  877. else:
  878. name = tree.children[0].value
  879. params = () # TODO terminal templates
  880. opts = int(tree.children[1]) if len(tree.children) == 3 else 1 # priority
  881. exp = tree.children[-1]
  882. if mangle is not None:
  883. params = tuple(mangle(p) for p in params)
  884. name = mangle(name)
  885. exp = _mangle_exp(exp, mangle)
  886. return name, exp, params, opts
  887. def load_grammar(self, grammar_text, grammar_name="<?>", mangle=None):
  888. tree = _parse_grammar(grammar_text, grammar_name)
  889. imports = {}
  890. for stmt in tree.children:
  891. if stmt.data == 'import':
  892. dotted_path, base_path, aliases = self._unpack_import(stmt, grammar_name)
  893. try:
  894. import_base_path, import_aliases = imports[dotted_path]
  895. assert base_path == import_base_path, 'Inconsistent base_path for %s.' % '.'.join(dotted_path)
  896. import_aliases.update(aliases)
  897. except KeyError:
  898. imports[dotted_path] = base_path, aliases
  899. for dotted_path, (base_path, aliases) in imports.items():
  900. self.do_import(dotted_path, base_path, aliases, mangle)
  901. for stmt in tree.children:
  902. if stmt.data in ('term', 'rule'):
  903. self._define(*self._unpack_definition(stmt, mangle))
  904. elif stmt.data == 'override':
  905. r ,= stmt.children
  906. self._define(*self._unpack_definition(r, mangle), override=True)
  907. elif stmt.data == 'extend':
  908. r ,= stmt.children
  909. self._extend(*self._unpack_definition(r, mangle))
  910. elif stmt.data == 'ignore':
  911. # if mangle is not None, we shouldn't apply ignore, since we aren't in a toplevel grammar
  912. if mangle is None:
  913. self._ignore(*stmt.children)
  914. elif stmt.data == 'declare':
  915. names = [t.value for t in stmt.children]
  916. if mangle is None:
  917. self._declare(*names)
  918. else:
  919. self._declare(*map(mangle, names))
  920. elif stmt.data == 'import':
  921. pass
  922. else:
  923. assert False, stmt
  924. term_defs = { name: exp
  925. for name, (_params, exp, _options) in self._definitions.items()
  926. if self._is_term(name)
  927. }
  928. resolve_term_references(term_defs)
  929. def _remove_unused(self, used):
  930. def rule_dependencies(symbol):
  931. if self._is_term(symbol):
  932. return []
  933. try:
  934. params, tree,_ = self._definitions[symbol]
  935. except KeyError:
  936. return []
  937. return _find_used_symbols(tree) - set(params)
  938. _used = set(bfs(used, rule_dependencies))
  939. self._definitions = {k: v for k, v in self._definitions.items() if k in _used}
  940. def do_import(self, dotted_path, base_path, aliases, base_mangle=None):
  941. assert dotted_path
  942. mangle = _get_mangle('__'.join(dotted_path), aliases, base_mangle)
  943. grammar_path = os.path.join(*dotted_path) + EXT
  944. to_try = self.import_paths + ([base_path] if base_path is not None else []) + [stdlib_loader]
  945. for source in to_try:
  946. try:
  947. if callable(source):
  948. joined_path, text = source(base_path, grammar_path)
  949. else:
  950. joined_path = os.path.join(source, grammar_path)
  951. with open(joined_path, encoding='utf8') as f:
  952. text = f.read()
  953. except IOError:
  954. continue
  955. else:
  956. gb = GrammarBuilder(self.global_keep_all_tokens, self.import_paths)
  957. gb.load_grammar(text, joined_path, mangle)
  958. gb._remove_unused(map(mangle, aliases))
  959. for name in gb._definitions:
  960. if name in self._definitions:
  961. raise GrammarError("Cannot import '%s' from '%s': Symbol already defined." % (name, grammar_path))
  962. self._definitions.update(**gb._definitions)
  963. break
  964. else:
  965. # Search failed. Make Python throw a nice error.
  966. open(grammar_path, encoding='utf8')
  967. assert False, "Couldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for it" % (dotted_path,)
  968. def validate(self):
  969. for name, (params, exp, _options) in self._definitions.items():
  970. for i, p in enumerate(params):
  971. if p in self._definitions:
  972. raise GrammarError("Template Parameter conflicts with rule %s (in template %s)" % (p, name))
  973. if p in params[:i]:
  974. raise GrammarError("Duplicate Template Parameter %s (in template %s)" % (p, name))
  975. if exp is None: # Remaining checks don't apply to abstract rules/terminals
  976. continue
  977. for temp in exp.find_data('template_usage'):
  978. sym = temp.children[0]
  979. args = temp.children[1:]
  980. if sym not in params:
  981. if sym not in self._definitions:
  982. self._grammar_error("Template '%s' used but not defined (in {type} {name})" % sym, name)
  983. if len(args) != len(self._definitions[sym][0]):
  984. expected, actual = len(self._definitions[sym][0]), len(args)
  985. self._grammar_error("Wrong number of template arguments used for {name} "
  986. "(expected %s, got %s) (in {type2} {name2})" % (expected, actual), sym, name)
  987. for sym in _find_used_symbols(exp):
  988. if sym not in self._definitions and sym not in params:
  989. self._grammar_error("{Type} '{name}' used but not defined (in {type2} {name2})", sym, name)
  990. if not set(self._definitions).issuperset(self._ignore_names):
  991. raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(self._ignore_names) - set(self._definitions)))
  992. def build(self):
  993. self.validate()
  994. rule_defs = []
  995. term_defs = []
  996. for name, (params, exp, options) in self._definitions.items():
  997. if self._is_term(name):
  998. assert len(params) == 0
  999. term_defs.append((name, (exp, options)))
  1000. else:
  1001. rule_defs.append((name, params, exp, options))
  1002. # resolve_term_references(term_defs)
  1003. return Grammar(rule_defs, term_defs, self._ignore_names)
  1004. def load_grammar(grammar, source, import_paths, global_keep_all_tokens):
  1005. builder = GrammarBuilder(global_keep_all_tokens, import_paths)
  1006. builder.load_grammar(grammar, source)
  1007. return builder.build()