This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

583 lines
24 KiB

  1. from __future__ import absolute_import
  2. from lark.exceptions import ConfigurationError, assert_config
  3. import sys, os, pickle, hashlib
  4. from io import open
  5. import tempfile
  6. from warnings import warn
  7. from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger, ABC, abstractmethod, verify_used_files
  8. from .load_grammar import load_grammar, FromPackageLoader, Grammar
  9. from .tree import Tree
  10. from .common import LexerConf, ParserConf
  11. from .lexer import Lexer, TraditionalLexer, TerminalDef, LexerThread
  12. from .parse_tree_builder import ParseTreeBuilder
  13. from .parser_frontends import get_frontend, _get_lexer_callbacks
  14. from .grammar import Rule
  15. import re
  16. try:
  17. import regex
  18. except ImportError:
  19. regex = None
  20. try:
  21. import atomicwrites
  22. except ImportError:
  23. atomicwrites = None
  24. ###{standalone
  25. class LarkOptions(Serialize):
  26. """Specifies the options for Lark
  27. """
  28. OPTIONS_DOC = """
  29. **=== General Options ===**
  30. start
  31. The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start")
  32. debug
  33. Display debug information and extra warnings. Use only when debugging (default: False)
  34. When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed.
  35. transformer
  36. Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster)
  37. propagate_positions
  38. Propagates (line, column, end_line, end_column) attributes into all tree branches.
  39. maybe_placeholders
  40. When True, the ``[]`` operator returns ``None`` when not matched.
  41. When ``False``, ``[]`` behaves like the ``?`` operator, and returns no value at all.
  42. (default= ``False``. Recommended to set to ``True``)
  43. cache
  44. Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now.
  45. - When ``False``, does nothing (default)
  46. - When ``True``, caches to a temporary file in the local directory
  47. - When given a string, caches to the path pointed by the string
  48. regex
  49. When True, uses the ``regex`` module instead of the stdlib ``re``.
  50. g_regex_flags
  51. Flags that are applied to all terminals (both regex and strings)
  52. keep_all_tokens
  53. Prevent the tree builder from automagically removing "punctuation" tokens (default: False)
  54. tree_class
  55. Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``.
  56. **=== Algorithm Options ===**
  57. parser
  58. Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley").
  59. (there is also a "cyk" option for legacy)
  60. lexer
  61. Decides whether or not to use a lexer stage
  62. - "auto" (default): Choose for me based on the parser
  63. - "standard": Use a standard lexer
  64. - "contextual": Stronger lexer (only works with parser="lalr")
  65. - "dynamic": Flexible and powerful (only with parser="earley")
  66. - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible.
  67. ambiguity
  68. Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
  69. - "resolve": The parser will automatically choose the simplest derivation
  70. (it chooses consistently: greedy for tokens, non-greedy for rules)
  71. - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
  72. - "forest": The parser will return the root of the shared packed parse forest.
  73. **=== Misc. / Domain Specific Options ===**
  74. postlex
  75. Lexer post-processing (Default: None) Only works with the standard and contextual lexers.
  76. priority
  77. How priorities should be evaluated - auto, none, normal, invert (Default: auto)
  78. lexer_callbacks
  79. Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
  80. use_bytes
  81. Accept an input of type ``bytes`` instead of ``str`` (Python 3 only).
  82. edit_terminals
  83. A callback for editing the terminals before parse.
  84. import_paths
  85. A List of either paths or loader functions to specify from where grammars are imported
  86. source_path
  87. Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading
  88. **=== End Options ===**
  89. """
  90. if __doc__:
  91. __doc__ += OPTIONS_DOC
  92. # Adding a new option needs to be done in multiple places:
  93. # - In the dictionary below. This is the primary truth of which options `Lark.__init__` accepts
  94. # - In the docstring above. It is used both for the docstring of `LarkOptions` and `Lark`, and in readthedocs
  95. # - In `lark-stubs/lark.pyi`:
  96. # - As attribute to `LarkOptions`
  97. # - As parameter to `Lark.__init__`
  98. # - Potentially in `_LOAD_ALLOWED_OPTIONS` below this class, when the option doesn't change how the grammar is loaded
  99. # - Potentially in `lark.tools.__init__`, if it makes sense, and it can easily be passed as a cmd argument
  100. _defaults = {
  101. 'debug': False,
  102. 'keep_all_tokens': False,
  103. 'tree_class': None,
  104. 'cache': False,
  105. 'postlex': None,
  106. 'parser': 'earley',
  107. 'lexer': 'auto',
  108. 'transformer': None,
  109. 'start': 'start',
  110. 'priority': 'auto',
  111. 'ambiguity': 'auto',
  112. 'regex': False,
  113. 'propagate_positions': False,
  114. 'lexer_callbacks': {},
  115. 'maybe_placeholders': False,
  116. 'edit_terminals': None,
  117. 'g_regex_flags': 0,
  118. 'use_bytes': False,
  119. 'import_paths': [],
  120. 'source_path': None,
  121. }
  122. def __init__(self, options_dict):
  123. o = dict(options_dict)
  124. options = {}
  125. for name, default in self._defaults.items():
  126. if name in o:
  127. value = o.pop(name)
  128. if isinstance(default, bool) and name not in ('cache', 'use_bytes'):
  129. value = bool(value)
  130. else:
  131. value = default
  132. options[name] = value
  133. if isinstance(options['start'], STRING_TYPE):
  134. options['start'] = [options['start']]
  135. self.__dict__['options'] = options
  136. assert_config(self.parser, ('earley', 'lalr', 'cyk', None))
  137. if self.parser == 'earley' and self.transformer:
  138. raise ConfigurationError('Cannot specify an embedded transformer when using the Earley algorithm.'
  139. 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)')
  140. if o:
  141. raise ConfigurationError("Unknown options: %s" % o.keys())
  142. def __getattr__(self, name):
  143. try:
  144. return self.__dict__['options'][name]
  145. except KeyError as e:
  146. raise AttributeError(e)
  147. def __setattr__(self, name, value):
  148. assert_config(name, self.options.keys(), "%r isn't a valid option. Expected one of: %s")
  149. self.options[name] = value
  150. def serialize(self, memo):
  151. return self.options
  152. @classmethod
  153. def deserialize(cls, data, memo):
  154. return cls(data)
  155. # Options that can be passed to the Lark parser, even when it was loaded from cache/standalone.
  156. # These option are only used outside of `load_grammar`.
  157. _LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
  158. _VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None)
  159. _VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest')
  160. class PostLex(ABC):
  161. @abstractmethod
  162. def process(self, stream):
  163. return stream
  164. always_accept = ()
  165. class Lark(Serialize):
  166. """Main interface for the library.
  167. It's mostly a thin wrapper for the many different parsers, and for the tree constructor.
  168. Parameters:
  169. grammar: a string or file-object containing the grammar spec (using Lark's ebnf syntax)
  170. options: a dictionary controlling various aspects of Lark.
  171. Example:
  172. >>> Lark(r'''start: "foo" ''')
  173. Lark(...)
  174. """
  175. def __init__(self, grammar, **options):
  176. self.options = LarkOptions(options)
  177. # Set regex or re module
  178. use_regex = self.options.regex
  179. if use_regex:
  180. if regex:
  181. re_module = regex
  182. else:
  183. raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.')
  184. else:
  185. re_module = re
  186. # Some, but not all file-like objects have a 'name' attribute
  187. if self.options.source_path is None:
  188. try:
  189. self.source_path = grammar.name
  190. except AttributeError:
  191. self.source_path = '<string>'
  192. else:
  193. self.source_path = self.options.source_path
  194. # Drain file-like objects to get their contents
  195. try:
  196. read = grammar.read
  197. except AttributeError:
  198. pass
  199. else:
  200. grammar = read()
  201. cache_fn = None
  202. cache_md5 = None
  203. if isinstance(grammar, STRING_TYPE):
  204. self.source_grammar = grammar
  205. if self.options.use_bytes:
  206. if not isascii(grammar):
  207. raise ConfigurationError("Grammar must be ascii only, when use_bytes=True")
  208. if sys.version_info[0] == 2 and self.options.use_bytes != 'force':
  209. raise ConfigurationError("`use_bytes=True` may have issues on python2."
  210. "Use `use_bytes='force'` to use it at your own risk.")
  211. if self.options.cache:
  212. if self.options.parser != 'lalr':
  213. raise ConfigurationError("cache only works with parser='lalr' for now")
  214. unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals')
  215. options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable)
  216. from . import __version__
  217. s = grammar + options_str + __version__ + str(sys.version_info[:2])
  218. cache_md5 = hashlib.md5(s.encode('utf8')).hexdigest()
  219. if isinstance(self.options.cache, STRING_TYPE):
  220. cache_fn = self.options.cache
  221. else:
  222. if self.options.cache is not True:
  223. raise ConfigurationError("cache argument must be bool or str")
  224. # Python2.7 doesn't support * syntax in tuples
  225. cache_fn = tempfile.gettempdir() + '/.lark_cache_%s_%s_%s.tmp' % ((cache_md5,) + sys.version_info[:2])
  226. if FS.exists(cache_fn):
  227. logger.debug('Loading grammar from cache: %s', cache_fn)
  228. # Remove options that aren't relevant for loading from cache
  229. for name in (set(options) - _LOAD_ALLOWED_OPTIONS):
  230. del options[name]
  231. with FS.open(cache_fn, 'rb') as f:
  232. file_md5 = f.readline().rstrip(b'\n')
  233. if file_md5 == cache_md5.encode('utf8') and verify_used_files(pickle.load(f)):
  234. old_options = self.options
  235. try:
  236. self._load(f, **options)
  237. except Exception: # We should probably narrow done which errors we catch here.
  238. logger.exception("Failed to load Lark from cache: %r. We will try to carry on." % cache_fn)
  239. # In theory, the Lark instance might have been messed up by the call to `_load`.
  240. # In practice the only relevant thing that might have been overriden should be `options`
  241. self.options = old_options
  242. else:
  243. return
  244. # Parse the grammar file and compose the grammars
  245. self.grammar, used_files = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens)
  246. else:
  247. assert isinstance(grammar, Grammar)
  248. self.grammar = grammar
  249. if self.options.lexer == 'auto':
  250. if self.options.parser == 'lalr':
  251. self.options.lexer = 'contextual'
  252. elif self.options.parser == 'earley':
  253. if self.options.postlex is not None:
  254. logger.info("postlex can't be used with the dynamic lexer, so we use standard instead. "
  255. "Consider using lalr with contextual instead of earley")
  256. self.options.lexer = 'standard'
  257. else:
  258. self.options.lexer = 'dynamic'
  259. elif self.options.parser == 'cyk':
  260. self.options.lexer = 'standard'
  261. else:
  262. assert False, self.options.parser
  263. lexer = self.options.lexer
  264. if isinstance(lexer, type):
  265. assert issubclass(lexer, Lexer) # XXX Is this really important? Maybe just ensure interface compliance
  266. else:
  267. assert_config(lexer, ('standard', 'contextual', 'dynamic', 'dynamic_complete'))
  268. if self.options.postlex is not None and 'dynamic' in lexer:
  269. raise ConfigurationError("Can't use postlex with a dynamic lexer. Use standard or contextual instead")
  270. if self.options.ambiguity == 'auto':
  271. if self.options.parser == 'earley':
  272. self.options.ambiguity = 'resolve'
  273. else:
  274. assert_config(self.options.parser, ('earley', 'cyk'), "%r doesn't support disambiguation. Use one of these parsers instead: %s")
  275. if self.options.priority == 'auto':
  276. self.options.priority = 'normal'
  277. if self.options.priority not in _VALID_PRIORITY_OPTIONS:
  278. raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS))
  279. assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"'
  280. if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS:
  281. raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS))
  282. if self.options.postlex is not None:
  283. terminals_to_keep = set(self.options.postlex.always_accept)
  284. else:
  285. terminals_to_keep = set()
  286. # Compile the EBNF grammar into BNF
  287. self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep)
  288. if self.options.edit_terminals:
  289. for t in self.terminals:
  290. self.options.edit_terminals(t)
  291. self._terminals_dict = {t.name: t for t in self.terminals}
  292. # If the user asked to invert the priorities, negate them all here.
  293. # This replaces the old 'resolve__antiscore_sum' option.
  294. if self.options.priority == 'invert':
  295. for rule in self.rules:
  296. if rule.options.priority is not None:
  297. rule.options.priority = -rule.options.priority
  298. # Else, if the user asked to disable priorities, strip them from the
  299. # rules. This allows the Earley parsers to skip an extra forest walk
  300. # for improved performance, if you don't need them (or didn't specify any).
  301. elif self.options.priority is None:
  302. for rule in self.rules:
  303. if rule.options.priority is not None:
  304. rule.options.priority = None
  305. # TODO Deprecate lexer_callbacks?
  306. self.lexer_conf = LexerConf(
  307. self.terminals, re_module, self.ignore_tokens, self.options.postlex,
  308. self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes
  309. )
  310. if self.options.parser:
  311. self.parser = self._build_parser()
  312. elif lexer:
  313. self.lexer = self._build_lexer()
  314. if cache_fn:
  315. logger.debug('Saving grammar to cache: %s', cache_fn)
  316. with FS.open(cache_fn, 'wb') as f:
  317. f.write(b'%s\n' % cache_md5.encode('utf8'))
  318. pickle.dump(used_files, f)
  319. self.save(f)
  320. if __doc__:
  321. __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC
  322. __serialize_fields__ = 'parser', 'rules', 'options'
  323. def _build_lexer(self, dont_ignore=False):
  324. lexer_conf = self.lexer_conf
  325. if dont_ignore:
  326. from copy import copy
  327. lexer_conf = copy(lexer_conf)
  328. lexer_conf.ignore = ()
  329. return TraditionalLexer(lexer_conf)
  330. def _prepare_callbacks(self):
  331. self._callbacks = {}
  332. # we don't need these callbacks if we aren't building a tree
  333. if self.options.ambiguity != 'forest':
  334. self._parse_tree_builder = ParseTreeBuilder(
  335. self.rules,
  336. self.options.tree_class or Tree,
  337. self.options.propagate_positions,
  338. self.options.parser != 'lalr' and self.options.ambiguity == 'explicit',
  339. self.options.maybe_placeholders
  340. )
  341. self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
  342. self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals))
  343. def _build_parser(self):
  344. self._prepare_callbacks()
  345. parser_class = get_frontend(self.options.parser, self.options.lexer)
  346. parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
  347. return parser_class(self.lexer_conf, parser_conf, options=self.options)
  348. def save(self, f):
  349. """Saves the instance into the given file object
  350. Useful for caching and multiprocessing.
  351. """
  352. data, m = self.memo_serialize([TerminalDef, Rule])
  353. pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL)
  354. @classmethod
  355. def load(cls, f):
  356. """Loads an instance from the given file object
  357. Useful for caching and multiprocessing.
  358. """
  359. inst = cls.__new__(cls)
  360. return inst._load(f)
  361. def _deserialize_lexer_conf(self, data, memo, options):
  362. lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo)
  363. lexer_conf.callbacks = options.lexer_callbacks or {}
  364. lexer_conf.re_module = regex if options.regex else re
  365. lexer_conf.use_bytes = options.use_bytes
  366. lexer_conf.g_regex_flags = options.g_regex_flags
  367. lexer_conf.skip_validation = True
  368. lexer_conf.postlex = options.postlex
  369. return lexer_conf
  370. def _load(self, f, **kwargs):
  371. if isinstance(f, dict):
  372. d = f
  373. else:
  374. d = pickle.load(f)
  375. memo = d['memo']
  376. data = d['data']
  377. assert memo
  378. memo = SerializeMemoizer.deserialize(memo, {'Rule': Rule, 'TerminalDef': TerminalDef}, {})
  379. options = dict(data['options'])
  380. if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults):
  381. raise ConfigurationError("Some options are not allowed when loading a Parser: {}"
  382. .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS))
  383. options.update(kwargs)
  384. self.options = LarkOptions.deserialize(options, memo)
  385. self.rules = [Rule.deserialize(r, memo) for r in data['rules']]
  386. self.source_path = '<deserialized>'
  387. parser_class = get_frontend(self.options.parser, self.options.lexer)
  388. self.lexer_conf = self._deserialize_lexer_conf(data['parser'], memo, self.options)
  389. self.terminals = self.lexer_conf.terminals
  390. self._prepare_callbacks()
  391. self._terminals_dict = {t.name: t for t in self.terminals}
  392. self.parser = parser_class.deserialize(
  393. data['parser'],
  394. memo,
  395. self.lexer_conf,
  396. self._callbacks,
  397. self.options, # Not all, but multiple attributes are used
  398. )
  399. return self
  400. @classmethod
  401. def _load_from_dict(cls, data, memo, **kwargs):
  402. inst = cls.__new__(cls)
  403. return inst._load({'data': data, 'memo': memo}, **kwargs)
  404. @classmethod
  405. def open(cls, grammar_filename, rel_to=None, **options):
  406. """Create an instance of Lark with the grammar given by its filename
  407. If ``rel_to`` is provided, the function will find the grammar filename in relation to it.
  408. Example:
  409. >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
  410. Lark(...)
  411. """
  412. if rel_to:
  413. basepath = os.path.dirname(rel_to)
  414. grammar_filename = os.path.join(basepath, grammar_filename)
  415. with open(grammar_filename, encoding='utf8') as f:
  416. return cls(f, **options)
  417. @classmethod
  418. def open_from_package(cls, package, grammar_path, search_paths=("",), **options):
  419. """Create an instance of Lark with the grammar loaded from within the package `package`.
  420. This allows grammar loading from zipapps.
  421. Imports in the grammar will use the `package` and `search_paths` provided, through `FromPackageLoader`
  422. Example:
  423. Lark.open_from_package(__name__, "example.lark", ("grammars",), parser=...)
  424. """
  425. package = FromPackageLoader(package, search_paths)
  426. full_path, text = package(None, grammar_path)
  427. options.setdefault('source_path', full_path)
  428. options.setdefault('import_paths', [])
  429. options['import_paths'].append(package)
  430. return cls(text, **options)
  431. def __repr__(self):
  432. return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer)
  433. def lex(self, text, dont_ignore=False):
  434. """Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'
  435. When dont_ignore=True, the lexer will return all tokens, even those marked for %ignore.
  436. """
  437. if not hasattr(self, 'lexer') or dont_ignore:
  438. lexer = self._build_lexer(dont_ignore)
  439. else:
  440. lexer = self.lexer
  441. lexer_thread = LexerThread(lexer, text)
  442. stream = lexer_thread.lex(None)
  443. if self.options.postlex:
  444. return self.options.postlex.process(stream)
  445. return stream
  446. def get_terminal(self, name):
  447. "Get information about a terminal"
  448. return self._terminals_dict[name]
  449. def parse(self, text, start=None, on_error=None):
  450. """Parse the given text, according to the options provided.
  451. Parameters:
  452. text (str): Text to be parsed.
  453. start (str, optional): Required if Lark was given multiple possible start symbols (using the start option).
  454. on_error (function, optional): if provided, will be called on UnexpectedToken error. Return true to resume parsing.
  455. LALR only. See examples/advanced/error_puppet.py for an example of how to use on_error.
  456. Returns:
  457. If a transformer is supplied to ``__init__``, returns whatever is the
  458. result of the transformation. Otherwise, returns a Tree instance.
  459. """
  460. return self.parser.parse(text, start=start, on_error=on_error)
  461. @property
  462. def source(self):
  463. warn("Lark.source attribute has been renamed to Lark.source_path", DeprecationWarning)
  464. return self.source_path
  465. @source.setter
  466. def source(self, value):
  467. self.source_path = value
  468. @property
  469. def grammar_source(self):
  470. warn("Lark.grammar_source attribute has been renamed to Lark.source_grammar", DeprecationWarning)
  471. return self.source_grammar
  472. @grammar_source.setter
  473. def grammar_source(self, value):
  474. self.source_grammar = value
  475. ###}