This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

601 lines
25 KiB

  1. from __future__ import absolute_import
  2. from lark.exceptions import ConfigurationError, assert_config
  3. import sys, os, pickle, hashlib
  4. from io import open
  5. import tempfile
  6. from warnings import warn
  7. from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger, ABC, abstractmethod
  8. from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files
  9. from .tree import Tree
  10. from .common import LexerConf, ParserConf
  11. from .lexer import Lexer, TraditionalLexer, TerminalDef, LexerThread
  12. from .parse_tree_builder import ParseTreeBuilder
  13. from .parser_frontends import get_frontend, _get_lexer_callbacks
  14. from .grammar import Rule
  15. import re
  16. try:
  17. import regex
  18. except ImportError:
  19. regex = None
  20. ###{standalone
  21. class LarkOptions(Serialize):
  22. """Specifies the options for Lark
  23. """
  24. OPTIONS_DOC = """
  25. **=== General Options ===**
  26. start
  27. The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start")
  28. debug
  29. Display debug information and extra warnings. Use only when debugging (default: False)
  30. When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed.
  31. transformer
  32. Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster)
  33. propagate_positions
  34. Propagates (line, column, end_line, end_column) attributes into all tree branches.
  35. Accepts ``False``, ``True``, or a callable, which will filter which nodes to ignore when propagating.
  36. maybe_placeholders
  37. When ``True``, the ``[]`` operator returns ``None`` when not matched.
  38. When ``False``, ``[]`` behaves like the ``?`` operator, and returns no value at all.
  39. (default= ``False``. Recommended to set to ``True``)
  40. cache
  41. Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now.
  42. - When ``False``, does nothing (default)
  43. - When ``True``, caches to a temporary file in the local directory
  44. - When given a string, caches to the path pointed by the string
  45. regex
  46. When True, uses the ``regex`` module instead of the stdlib ``re``.
  47. g_regex_flags
  48. Flags that are applied to all terminals (both regex and strings)
  49. keep_all_tokens
  50. Prevent the tree builder from automagically removing "punctuation" tokens (default: False)
  51. tree_class
  52. Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``.
  53. **=== Algorithm Options ===**
  54. parser
  55. Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley").
  56. (there is also a "cyk" option for legacy)
  57. lexer
  58. Decides whether or not to use a lexer stage
  59. - "auto" (default): Choose for me based on the parser
  60. - "standard": Use a standard lexer
  61. - "contextual": Stronger lexer (only works with parser="lalr")
  62. - "dynamic": Flexible and powerful (only with parser="earley")
  63. - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible.
  64. ambiguity
  65. Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
  66. - "resolve": The parser will automatically choose the simplest derivation
  67. (it chooses consistently: greedy for tokens, non-greedy for rules)
  68. - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
  69. - "forest": The parser will return the root of the shared packed parse forest.
  70. **=== Misc. / Domain Specific Options ===**
  71. postlex
  72. Lexer post-processing (Default: None) Only works with the standard and contextual lexers.
  73. priority
  74. How priorities should be evaluated - auto, none, normal, invert (Default: auto)
  75. lexer_callbacks
  76. Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
  77. use_bytes
  78. Accept an input of type ``bytes`` instead of ``str`` (Python 3 only).
  79. edit_terminals
  80. A callback for editing the terminals before parse.
  81. import_paths
  82. A List of either paths or loader functions to specify from where grammars are imported
  83. source_path
  84. Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading
  85. **=== End of Options ===**
  86. """
  87. if __doc__:
  88. __doc__ += OPTIONS_DOC
  89. # Adding a new option needs to be done in multiple places:
  90. # - In the dictionary below. This is the primary truth of which options `Lark.__init__` accepts
  91. # - In the docstring above. It is used both for the docstring of `LarkOptions` and `Lark`, and in readthedocs
  92. # - In `lark-stubs/lark.pyi`:
  93. # - As attribute to `LarkOptions`
  94. # - As parameter to `Lark.__init__`
  95. # - Potentially in `_LOAD_ALLOWED_OPTIONS` below this class, when the option doesn't change how the grammar is loaded
  96. # - Potentially in `lark.tools.__init__`, if it makes sense, and it can easily be passed as a cmd argument
  97. _defaults = {
  98. 'debug': False,
  99. 'keep_all_tokens': False,
  100. 'tree_class': None,
  101. 'cache': False,
  102. 'postlex': None,
  103. 'parser': 'earley',
  104. 'lexer': 'auto',
  105. 'transformer': None,
  106. 'start': 'start',
  107. 'priority': 'auto',
  108. 'ambiguity': 'auto',
  109. 'regex': False,
  110. 'propagate_positions': False,
  111. 'lexer_callbacks': {},
  112. 'maybe_placeholders': False,
  113. 'edit_terminals': None,
  114. 'g_regex_flags': 0,
  115. 'use_bytes': False,
  116. 'import_paths': [],
  117. 'source_path': None,
  118. }
  119. def __init__(self, options_dict):
  120. o = dict(options_dict)
  121. options = {}
  122. for name, default in self._defaults.items():
  123. if name in o:
  124. value = o.pop(name)
  125. if isinstance(default, bool) and name not in ('cache', 'use_bytes', 'propagate_positions'):
  126. value = bool(value)
  127. else:
  128. value = default
  129. options[name] = value
  130. if isinstance(options['start'], STRING_TYPE):
  131. options['start'] = [options['start']]
  132. self.__dict__['options'] = options
  133. assert_config(self.parser, ('earley', 'lalr', 'cyk', None))
  134. if self.parser == 'earley' and self.transformer:
  135. raise ConfigurationError('Cannot specify an embedded transformer when using the Earley algorithm. '
  136. 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)')
  137. if o:
  138. raise ConfigurationError("Unknown options: %s" % o.keys())
  139. def __getattr__(self, name):
  140. try:
  141. return self.__dict__['options'][name]
  142. except KeyError as e:
  143. raise AttributeError(e)
  144. def __setattr__(self, name, value):
  145. assert_config(name, self.options.keys(), "%r isn't a valid option. Expected one of: %s")
  146. self.options[name] = value
  147. def serialize(self, memo):
  148. return self.options
  149. @classmethod
  150. def deserialize(cls, data, memo):
  151. return cls(data)
  152. # Options that can be passed to the Lark parser, even when it was loaded from cache/standalone.
  153. # These option are only used outside of `load_grammar`.
  154. _LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
  155. _VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None)
  156. _VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest')
  157. class PostLex(ABC):
  158. @abstractmethod
  159. def process(self, stream):
  160. return stream
  161. always_accept = ()
  162. class Lark(Serialize):
  163. """Main interface for the library.
  164. It's mostly a thin wrapper for the many different parsers, and for the tree constructor.
  165. Parameters:
  166. grammar: a string or file-object containing the grammar spec (using Lark's ebnf syntax)
  167. options: a dictionary controlling various aspects of Lark.
  168. Example:
  169. >>> Lark(r'''start: "foo" ''')
  170. Lark(...)
  171. """
  172. def __init__(self, grammar, **options):
  173. self.options = LarkOptions(options)
  174. # Set regex or re module
  175. use_regex = self.options.regex
  176. if use_regex:
  177. if regex:
  178. re_module = regex
  179. else:
  180. raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.')
  181. else:
  182. re_module = re
  183. # Some, but not all file-like objects have a 'name' attribute
  184. if self.options.source_path is None:
  185. try:
  186. self.source_path = grammar.name
  187. except AttributeError:
  188. self.source_path = '<string>'
  189. else:
  190. self.source_path = self.options.source_path
  191. # Drain file-like objects to get their contents
  192. try:
  193. read = grammar.read
  194. except AttributeError:
  195. pass
  196. else:
  197. grammar = read()
  198. cache_fn = None
  199. cache_md5 = None
  200. if isinstance(grammar, STRING_TYPE):
  201. self.source_grammar = grammar
  202. if self.options.use_bytes:
  203. if not isascii(grammar):
  204. raise ConfigurationError("Grammar must be ascii only, when use_bytes=True")
  205. if sys.version_info[0] == 2 and self.options.use_bytes != 'force':
  206. raise ConfigurationError("`use_bytes=True` may have issues on python2."
  207. "Use `use_bytes='force'` to use it at your own risk.")
  208. if self.options.cache:
  209. if self.options.parser != 'lalr':
  210. raise ConfigurationError("cache only works with parser='lalr' for now")
  211. unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals')
  212. options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable)
  213. from . import __version__
  214. s = grammar + options_str + __version__ + str(sys.version_info[:2])
  215. cache_md5 = hashlib.md5(s.encode('utf8')).hexdigest()
  216. if isinstance(self.options.cache, STRING_TYPE):
  217. cache_fn = self.options.cache
  218. else:
  219. if self.options.cache is not True:
  220. raise ConfigurationError("cache argument must be bool or str")
  221. # Python2.7 doesn't support * syntax in tuples
  222. cache_fn = tempfile.gettempdir() + '/.lark_cache_%s_%s_%s.tmp' % ((cache_md5,) + sys.version_info[:2])
  223. if FS.exists(cache_fn):
  224. logger.debug('Loading grammar from cache: %s', cache_fn)
  225. # Remove options that aren't relevant for loading from cache
  226. for name in (set(options) - _LOAD_ALLOWED_OPTIONS):
  227. del options[name]
  228. with FS.open(cache_fn, 'rb') as f:
  229. old_options = self.options
  230. try:
  231. file_md5 = f.readline().rstrip(b'\n')
  232. cached_used_files = pickle.load(f)
  233. if file_md5 == cache_md5.encode('utf8') and verify_used_files(cached_used_files):
  234. cached_parser_data = pickle.load(f)
  235. self._load(cached_parser_data, **options)
  236. return
  237. except Exception: # We should probably narrow done which errors we catch here.
  238. logger.exception("Failed to load Lark from cache: %r. We will try to carry on." % cache_fn)
  239. # In theory, the Lark instance might have been messed up by the call to `_load`.
  240. # In practice the only relevant thing that might have been overriden should be `options`
  241. self.options = old_options
  242. # Parse the grammar file and compose the grammars
  243. self.grammar, used_files = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens)
  244. else:
  245. assert isinstance(grammar, Grammar)
  246. self.grammar = grammar
  247. if self.options.lexer == 'auto':
  248. if self.options.parser == 'lalr':
  249. self.options.lexer = 'contextual'
  250. elif self.options.parser == 'earley':
  251. if self.options.postlex is not None:
  252. logger.info("postlex can't be used with the dynamic lexer, so we use standard instead. "
  253. "Consider using lalr with contextual instead of earley")
  254. self.options.lexer = 'standard'
  255. else:
  256. self.options.lexer = 'dynamic'
  257. elif self.options.parser == 'cyk':
  258. self.options.lexer = 'standard'
  259. else:
  260. assert False, self.options.parser
  261. lexer = self.options.lexer
  262. if isinstance(lexer, type):
  263. assert issubclass(lexer, Lexer) # XXX Is this really important? Maybe just ensure interface compliance
  264. else:
  265. assert_config(lexer, ('standard', 'contextual', 'dynamic', 'dynamic_complete'))
  266. if self.options.postlex is not None and 'dynamic' in lexer:
  267. raise ConfigurationError("Can't use postlex with a dynamic lexer. Use standard or contextual instead")
  268. if self.options.ambiguity == 'auto':
  269. if self.options.parser == 'earley':
  270. self.options.ambiguity = 'resolve'
  271. else:
  272. assert_config(self.options.parser, ('earley', 'cyk'), "%r doesn't support disambiguation. Use one of these parsers instead: %s")
  273. if self.options.priority == 'auto':
  274. self.options.priority = 'normal'
  275. if self.options.priority not in _VALID_PRIORITY_OPTIONS:
  276. raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS))
  277. assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"'
  278. if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS:
  279. raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS))
  280. if self.options.postlex is not None:
  281. terminals_to_keep = set(self.options.postlex.always_accept)
  282. else:
  283. terminals_to_keep = set()
  284. # Compile the EBNF grammar into BNF
  285. self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep)
  286. if self.options.edit_terminals:
  287. for t in self.terminals:
  288. self.options.edit_terminals(t)
  289. self._terminals_dict = {t.name: t for t in self.terminals}
  290. # If the user asked to invert the priorities, negate them all here.
  291. # This replaces the old 'resolve__antiscore_sum' option.
  292. if self.options.priority == 'invert':
  293. for rule in self.rules:
  294. if rule.options.priority is not None:
  295. rule.options.priority = -rule.options.priority
  296. # Else, if the user asked to disable priorities, strip them from the
  297. # rules. This allows the Earley parsers to skip an extra forest walk
  298. # for improved performance, if you don't need them (or didn't specify any).
  299. elif self.options.priority is None:
  300. for rule in self.rules:
  301. if rule.options.priority is not None:
  302. rule.options.priority = None
  303. # TODO Deprecate lexer_callbacks?
  304. self.lexer_conf = LexerConf(
  305. self.terminals, re_module, self.ignore_tokens, self.options.postlex,
  306. self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes
  307. )
  308. if self.options.parser:
  309. self.parser = self._build_parser()
  310. elif lexer:
  311. self.lexer = self._build_lexer()
  312. if cache_fn:
  313. logger.debug('Saving grammar to cache: %s', cache_fn)
  314. with FS.open(cache_fn, 'wb') as f:
  315. f.write(cache_md5.encode('utf8') + b'\n')
  316. pickle.dump(used_files, f)
  317. self.save(f)
  318. if __doc__:
  319. __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC
  320. __serialize_fields__ = 'parser', 'rules', 'options'
  321. def _build_lexer(self, dont_ignore=False):
  322. lexer_conf = self.lexer_conf
  323. if dont_ignore:
  324. from copy import copy
  325. lexer_conf = copy(lexer_conf)
  326. lexer_conf.ignore = ()
  327. return TraditionalLexer(lexer_conf)
  328. def _prepare_callbacks(self):
  329. self._callbacks = {}
  330. # we don't need these callbacks if we aren't building a tree
  331. if self.options.ambiguity != 'forest':
  332. self._parse_tree_builder = ParseTreeBuilder(
  333. self.rules,
  334. self.options.tree_class or Tree,
  335. self.options.propagate_positions,
  336. self.options.parser != 'lalr' and self.options.ambiguity == 'explicit',
  337. self.options.maybe_placeholders
  338. )
  339. self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
  340. self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals))
  341. def _build_parser(self):
  342. self._prepare_callbacks()
  343. parser_class = get_frontend(self.options.parser, self.options.lexer)
  344. parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
  345. return parser_class(self.lexer_conf, parser_conf, options=self.options)
  346. def save(self, f):
  347. """Saves the instance into the given file object
  348. Useful for caching and multiprocessing.
  349. """
  350. data, m = self.memo_serialize([TerminalDef, Rule])
  351. pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL)
  352. @classmethod
  353. def load(cls, f):
  354. """Loads an instance from the given file object
  355. Useful for caching and multiprocessing.
  356. """
  357. inst = cls.__new__(cls)
  358. return inst._load(f)
  359. def _deserialize_lexer_conf(self, data, memo, options):
  360. lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo)
  361. lexer_conf.callbacks = options.lexer_callbacks or {}
  362. lexer_conf.re_module = regex if options.regex else re
  363. lexer_conf.use_bytes = options.use_bytes
  364. lexer_conf.g_regex_flags = options.g_regex_flags
  365. lexer_conf.skip_validation = True
  366. lexer_conf.postlex = options.postlex
  367. return lexer_conf
  368. def _load(self, f, **kwargs):
  369. if isinstance(f, dict):
  370. d = f
  371. else:
  372. d = pickle.load(f)
  373. memo_json = d['memo']
  374. data = d['data']
  375. assert memo_json
  376. memo = SerializeMemoizer.deserialize(memo_json, {'Rule': Rule, 'TerminalDef': TerminalDef}, {})
  377. options = dict(data['options'])
  378. if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults):
  379. raise ConfigurationError("Some options are not allowed when loading a Parser: {}"
  380. .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS))
  381. options.update(kwargs)
  382. self.options = LarkOptions.deserialize(options, memo)
  383. self.rules = [Rule.deserialize(r, memo) for r in data['rules']]
  384. self.source_path = '<deserialized>'
  385. parser_class = get_frontend(self.options.parser, self.options.lexer)
  386. self.lexer_conf = self._deserialize_lexer_conf(data['parser'], memo, self.options)
  387. self.terminals = self.lexer_conf.terminals
  388. self._prepare_callbacks()
  389. self._terminals_dict = {t.name: t for t in self.terminals}
  390. self.parser = parser_class.deserialize(
  391. data['parser'],
  392. memo,
  393. self.lexer_conf,
  394. self._callbacks,
  395. self.options, # Not all, but multiple attributes are used
  396. )
  397. return self
  398. @classmethod
  399. def _load_from_dict(cls, data, memo, **kwargs):
  400. inst = cls.__new__(cls)
  401. return inst._load({'data': data, 'memo': memo}, **kwargs)
  402. @classmethod
  403. def open(cls, grammar_filename, rel_to=None, **options):
  404. """Create an instance of Lark with the grammar given by its filename
  405. If ``rel_to`` is provided, the function will find the grammar filename in relation to it.
  406. Example:
  407. >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
  408. Lark(...)
  409. """
  410. if rel_to:
  411. basepath = os.path.dirname(rel_to)
  412. grammar_filename = os.path.join(basepath, grammar_filename)
  413. with open(grammar_filename, encoding='utf8') as f:
  414. return cls(f, **options)
  415. @classmethod
  416. def open_from_package(cls, package, grammar_path, search_paths=("",), **options):
  417. """Create an instance of Lark with the grammar loaded from within the package `package`.
  418. This allows grammar loading from zipapps.
  419. Imports in the grammar will use the `package` and `search_paths` provided, through `FromPackageLoader`
  420. Example:
  421. Lark.open_from_package(__name__, "example.lark", ("grammars",), parser=...)
  422. """
  423. package_loader = FromPackageLoader(package, search_paths)
  424. full_path, text = package_loader(None, grammar_path)
  425. options.setdefault('source_path', full_path)
  426. options.setdefault('import_paths', [])
  427. options['import_paths'].append(package_loader)
  428. return cls(text, **options)
  429. def __repr__(self):
  430. return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer)
  431. def lex(self, text, dont_ignore=False):
  432. """Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'
  433. When dont_ignore=True, the lexer will return all tokens, even those marked for %ignore.
  434. :raises UnexpectedCharacters: In case the lexer cannot find a suitable match.
  435. """
  436. if not hasattr(self, 'lexer') or dont_ignore:
  437. lexer = self._build_lexer(dont_ignore)
  438. else:
  439. lexer = self.lexer
  440. lexer_thread = LexerThread(lexer, text)
  441. stream = lexer_thread.lex(None)
  442. if self.options.postlex:
  443. return self.options.postlex.process(stream)
  444. return stream
  445. def get_terminal(self, name):
  446. """Get information about a terminal"""
  447. return self._terminals_dict[name]
  448. def parse_interactive(self, text=None, start=None):
  449. """Start an interactive parsing session.
  450. Parameters:
  451. text (str, optional): Text to be parsed. Required for ``resume_parse()``.
  452. start (str, optional): Start symbol
  453. Returns:
  454. A new InteractiveParser instance.
  455. See Also: ``Lark.parse()``
  456. """
  457. return self.parser.parse_interactive(text, start=start)
  458. def parse(self, text, start=None, on_error=None):
  459. """Parse the given text, according to the options provided.
  460. Parameters:
  461. text (str): Text to be parsed.
  462. start (str, optional): Required if Lark was given multiple possible start symbols (using the start option).
  463. on_error (function, optional): if provided, will be called on UnexpectedToken error. Return true to resume parsing.
  464. LALR only. See examples/advanced/error_handling.py for an example of how to use on_error.
  465. Returns:
  466. If a transformer is supplied to ``__init__``, returns whatever is the
  467. result of the transformation. Otherwise, returns a Tree instance.
  468. :raises UnexpectedInput: On a parse error, one of these sub-exceptions will rise:
  469. ``UnexpectedCharacters``, ``UnexpectedToken``, or ``UnexpectedEOF``.
  470. For convenience, these sub-exceptions also inherit from ``ParserError`` and ``LexerError``.
  471. """
  472. return self.parser.parse(text, start=start, on_error=on_error)
  473. @property
  474. def source(self):
  475. warn("Attribute Lark.source was renamed to Lark.source_path", DeprecationWarning)
  476. return self.source_path
  477. @source.setter
  478. def source(self, value):
  479. self.source_path = value
  480. @property
  481. def grammar_source(self):
  482. warn("Attribute Lark.grammar_source was renamed to Lark.source_grammar", DeprecationWarning)
  483. return self.source_grammar
  484. @grammar_source.setter
  485. def grammar_source(self, value):
  486. self.source_grammar = value
  487. ###}