This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

211 lines
8.2 KiB

  1. from __future__ import absolute_import
  2. import os
  3. import time
  4. from collections import defaultdict
  5. from .utils import STRING_TYPE
  6. from .load_grammar import load_grammar
  7. from .tree import Tree
  8. from .common import LexerConf, ParserConf
  9. from .lexer import Lexer
  10. from .parse_tree_builder import ParseTreeBuilder
  11. from .parser_frontends import get_frontend
  12. class LarkOptions(object):
  13. """Specifies the options for Lark
  14. """
  15. OPTIONS_DOC = """
  16. parser - Decides which parser engine to use, "earley" or "lalr". (Default: "earley")
  17. Note: "lalr" requires a lexer
  18. lexer - Decides whether or not to use a lexer stage
  19. None: Don't use a lexer (scanless, only works with parser="earley")
  20. "standard": Use a standard lexer
  21. "contextual": Stronger lexer (only works with parser="lalr")
  22. "auto" (default): Choose for me based on grammar and parser
  23. ambiguity - Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
  24. "resolve": The parser will automatically choose the simplest derivation
  25. (it chooses consistently: greedy for tokens, non-greedy for rules)
  26. "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
  27. transformer - Applies the transformer to every parse tree
  28. debug - Affects verbosity (default: False)
  29. keep_all_tokens - Don't automagically remove "punctuation" tokens (default: False)
  30. cache_grammar - Cache the Lark grammar (Default: False)
  31. postlex - Lexer post-processing (Requires standard lexer. Default: None)
  32. start - The start symbol (Default: start)
  33. profile - Measure run-time usage in Lark. Read results from the profiler proprety (Default: False)
  34. propagate_positions - Propagates [line, column, end_line, end_column] attributes into all tree branches.
  35. lexer_callbacks - Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
  36. """
  37. __doc__ += OPTIONS_DOC
  38. def __init__(self, options_dict):
  39. o = dict(options_dict)
  40. self.debug = bool(o.pop('debug', False))
  41. self.keep_all_tokens = bool(o.pop('keep_all_tokens', False))
  42. self.tree_class = o.pop('tree_class', Tree)
  43. self.cache_grammar = o.pop('cache_grammar', False)
  44. self.postlex = o.pop('postlex', None)
  45. self.parser = o.pop('parser', 'earley')
  46. self.lexer = o.pop('lexer', 'auto')
  47. self.transformer = o.pop('transformer', None)
  48. self.start = o.pop('start', 'start')
  49. self.profile = o.pop('profile', False)
  50. self.ambiguity = o.pop('ambiguity', 'auto')
  51. self.propagate_positions = o.pop('propagate_positions', False)
  52. self.earley__predict_all = o.pop('earley__predict_all', False)
  53. self.lexer_callbacks = o.pop('lexer_callbacks', {})
  54. assert self.parser in ('earley', 'lalr', 'cyk', None)
  55. if self.parser == 'earley' and self.transformer:
  56. raise ValueError('Cannot specify an embedded transformer when using the Earley algorithm.'
  57. 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. lalr)')
  58. if o:
  59. raise ValueError("Unknown options: %s" % o.keys())
  60. class Profiler:
  61. def __init__(self):
  62. self.total_time = defaultdict(float)
  63. self.cur_section = '__init__'
  64. self.last_enter_time = time.time()
  65. def enter_section(self, name):
  66. cur_time = time.time()
  67. self.total_time[self.cur_section] += cur_time - self.last_enter_time
  68. self.last_enter_time = cur_time
  69. self.cur_section = name
  70. def make_wrapper(self, name, f):
  71. def wrapper(*args, **kwargs):
  72. last_section = self.cur_section
  73. self.enter_section(name)
  74. try:
  75. return f(*args, **kwargs)
  76. finally:
  77. self.enter_section(last_section)
  78. return wrapper
  79. class Lark:
  80. def __init__(self, grammar, **options):
  81. """
  82. grammar : a string or file-object containing the grammar spec (using Lark's ebnf syntax)
  83. options : a dictionary controlling various aspects of Lark.
  84. """
  85. self.options = LarkOptions(options)
  86. # Some, but not all file-like objects have a 'name' attribute
  87. try:
  88. source = grammar.name
  89. except AttributeError:
  90. source = '<string>'
  91. cache_file = "larkcache_%s" % str(hash(grammar)%(2**32))
  92. else:
  93. cache_file = "larkcache_%s" % os.path.basename(source)
  94. # Drain file-like objects to get their contents
  95. try:
  96. read = grammar.read
  97. except AttributeError:
  98. pass
  99. else:
  100. grammar = read()
  101. assert isinstance(grammar, STRING_TYPE)
  102. if self.options.cache_grammar:
  103. raise NotImplementedError("Not available yet")
  104. assert not self.options.profile, "Feature temporarily disabled"
  105. self.profiler = Profiler() if self.options.profile else None
  106. if self.options.lexer == 'auto':
  107. if self.options.parser == 'lalr':
  108. self.options.lexer = 'standard'
  109. elif self.options.parser == 'earley':
  110. self.options.lexer = 'dynamic'
  111. elif self.options.parser == 'cyk':
  112. self.options.lexer = 'standard'
  113. else:
  114. assert False, self.options.parser
  115. lexer = self.options.lexer
  116. assert lexer in ('standard', 'contextual', 'dynamic', None)
  117. if self.options.ambiguity == 'auto':
  118. if self.options.parser == 'earley':
  119. self.options.ambiguity = 'resolve'
  120. else:
  121. disambig_parsers = ['earley', 'cyk']
  122. assert self.options.parser in disambig_parsers, (
  123. 'Only %s supports disambiguation right now') % ', '.join(disambig_parsers)
  124. assert self.options.ambiguity in ('resolve', 'explicit', 'auto', 'resolve__antiscore_sum')
  125. # Parse the grammar file and compose the grammars (TODO)
  126. self.grammar = load_grammar(grammar, source)
  127. # Compile the EBNF grammar into BNF
  128. tokens, self.rules, self.ignore_tokens = self.grammar.compile(lexer=bool(lexer), start=self.options.start)
  129. self.lexer_conf = LexerConf(tokens, self.ignore_tokens, self.options.postlex, self.options.lexer_callbacks)
  130. if self.options.parser:
  131. self.parser = self._build_parser()
  132. elif lexer:
  133. self.lexer = self._build_lexer()
  134. if self.profiler: self.profiler.enter_section('outside_lark')
  135. __init__.__doc__ += "\nOPTIONS:" + LarkOptions.OPTIONS_DOC
  136. def _build_lexer(self):
  137. return Lexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks)
  138. def _build_parser(self):
  139. self.parser_class = get_frontend(self.options.parser, self.options.lexer)
  140. self._parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class, self.options.propagate_positions, self.options.keep_all_tokens)
  141. callback = self._parse_tree_builder.create_callback(self.options.transformer)
  142. if self.profiler:
  143. for f in dir(callback):
  144. if not (f.startswith('__') and f.endswith('__')):
  145. setattr(callback, f, self.profiler.make_wrapper('transformer', getattr(callback, f)))
  146. parser_conf = ParserConf(self.rules, callback, self.options.start)
  147. return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
  148. def lex(self, text):
  149. if not hasattr(self, 'lexer'):
  150. self.lexer = self._build_lexer()
  151. stream = self.lexer.lex(text)
  152. if self.options.postlex:
  153. return self.options.postlex.process(stream)
  154. else:
  155. return stream
  156. def parse(self, text):
  157. return self.parser.parse(text)
  158. # if self.profiler:
  159. # self.profiler.enter_section('lex')
  160. # l = list(self.lex(text))
  161. # self.profiler.enter_section('parse')
  162. # try:
  163. # return self.parser.parse(l)
  164. # finally:
  165. # self.profiler.enter_section('outside_lark')
  166. # else:
  167. # l = list(self.lex(text))
  168. # return self.parser.parse(l)