This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

207 lines
8.1 KiB

  1. from __future__ import absolute_import
  2. import os
  3. import time
  4. from collections import defaultdict
  5. from .utils import STRING_TYPE
  6. from .load_grammar import load_grammar
  7. from .tree import Tree
  8. from .common import LexerConf, ParserConf
  9. from .lexer import Lexer
  10. from .parse_tree_builder import ParseTreeBuilder
  11. from .parser_frontends import get_frontend
  12. class LarkOptions(object):
  13. """Specifies the options for Lark
  14. """
  15. OPTIONS_DOC = """
  16. parser - Decides which parser engine to use, "earley" or "lalr". (Default: "earley")
  17. Note: "lalr" requires a lexer
  18. lexer - Decides whether or not to use a lexer stage
  19. None: Don't use a lexer (scanless, only works with parser="earley")
  20. "standard": Use a standard lexer
  21. "contextual": Stronger lexer (only works with parser="lalr")
  22. "auto" (default): Choose for me based on grammar and parser
  23. ambiguity - Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
  24. "resolve": The parser will automatically choose the simplest derivation
  25. (it chooses consistently: greedy for tokens, non-greedy for rules)
  26. "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
  27. earley__all_derivations - If True, try every possible derivation of each rule. If False, pick the first
  28. correct derivation. Both will find a solution to every correct grammar & input,
  29. but when False, some ambiguities won't appear (Default: True)
  30. transformer - Applies the transformer to every parse tree
  31. debug - Affects verbosity (default: False)
  32. keep_all_tokens - Don't automagically remove "punctuation" tokens (default: False)
  33. cache_grammar - Cache the Lark grammar (Default: False)
  34. postlex - Lexer post-processing (Requires standard lexer. Default: None)
  35. start - The start symbol (Default: start)
  36. profile - Measure run-time usage in Lark. Read results from the profiler proprety (Default: False)
  37. propagate_positions - Experimental. Don't use yet.
  38. """
  39. __doc__ += OPTIONS_DOC
  40. def __init__(self, options_dict):
  41. o = dict(options_dict)
  42. self.debug = bool(o.pop('debug', False))
  43. self.keep_all_tokens = bool(o.pop('keep_all_tokens', False))
  44. self.tree_class = o.pop('tree_class', Tree)
  45. self.cache_grammar = o.pop('cache_grammar', False)
  46. self.postlex = o.pop('postlex', None)
  47. self.parser = o.pop('parser', 'earley')
  48. self.lexer = o.pop('lexer', 'auto')
  49. self.transformer = o.pop('transformer', None)
  50. self.start = o.pop('start', 'start')
  51. self.profile = o.pop('profile', False)
  52. self.ambiguity = o.pop('ambiguity', 'auto')
  53. self.propagate_positions = o.pop('propagate_positions', False)
  54. self.earley__all_derivations = o.pop('earley__all_derivations', True)
  55. assert self.parser in ('earley', 'lalr', None)
  56. if self.parser == 'earley' and self.transformer:
  57. raise ValueError('Cannot specify an embedded transformer when using the Earley algorithm.'
  58. 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. lalr)')
  59. if o:
  60. raise ValueError("Unknown options: %s" % o.keys())
  61. class Profiler:
  62. def __init__(self):
  63. self.total_time = defaultdict(float)
  64. self.cur_section = '__init__'
  65. self.last_enter_time = time.time()
  66. def enter_section(self, name):
  67. cur_time = time.time()
  68. self.total_time[self.cur_section] += cur_time - self.last_enter_time
  69. self.last_enter_time = cur_time
  70. self.cur_section = name
  71. def make_wrapper(self, name, f):
  72. def wrapper(*args, **kwargs):
  73. last_section = self.cur_section
  74. self.enter_section(name)
  75. try:
  76. return f(*args, **kwargs)
  77. finally:
  78. self.enter_section(last_section)
  79. return wrapper
  80. class Lark:
  81. def __init__(self, grammar, **options):
  82. """
  83. grammar : a string or file-object containing the grammar spec (using Lark's ebnf syntax)
  84. options : a dictionary controlling various aspects of Lark.
  85. """
  86. self.options = LarkOptions(options)
  87. # Some, but not all file-like objects have a 'name' attribute
  88. try:
  89. source = grammar.name
  90. except AttributeError:
  91. source = '<string>'
  92. cache_file = "larkcache_%s" % str(hash(grammar)%(2**32))
  93. else:
  94. cache_file = "larkcache_%s" % os.path.basename(source)
  95. # Drain file-like objects to get their contents
  96. try:
  97. read = grammar.read
  98. except AttributeError:
  99. pass
  100. else:
  101. grammar = read()
  102. assert isinstance(grammar, STRING_TYPE)
  103. if self.options.cache_grammar:
  104. raise NotImplementedError("Not available yet")
  105. assert not self.options.profile, "Feature temporarily disabled"
  106. self.profiler = Profiler() if self.options.profile else None
  107. if self.options.lexer == 'auto':
  108. if self.options.parser == 'lalr':
  109. self.options.lexer = 'standard'
  110. elif self.options.parser == 'earley':
  111. self.options.lexer = 'dynamic'
  112. else:
  113. assert False, self.options.parser
  114. lexer = self.options.lexer
  115. assert lexer in ('standard', 'contextual', 'dynamic', None)
  116. if self.options.ambiguity == 'auto':
  117. if self.options.parser == 'earley':
  118. self.options.ambiguity = 'resolve'
  119. else:
  120. assert self.options.parser == 'earley', "Only Earley supports disambiguation right now"
  121. assert self.options.ambiguity in ('resolve', 'explicit', 'auto', 'resolve__antiscore_sum')
  122. # Parse the grammar file and compose the grammars (TODO)
  123. self.grammar = load_grammar(grammar, source)
  124. # Compile the EBNF grammar into BNF
  125. tokens, self.rules, self.ignore_tokens = self.grammar.compile(lexer=bool(lexer), start=self.options.start)
  126. self.lexer_conf = LexerConf(tokens, self.ignore_tokens, self.options.postlex)
  127. if self.options.parser:
  128. self.parser = self._build_parser()
  129. elif lexer:
  130. self.lexer = self._build_lexer()
  131. if self.profiler: self.profiler.enter_section('outside_lark')
  132. __init__.__doc__ += "\nOPTIONS:" + LarkOptions.OPTIONS_DOC
  133. def _build_lexer(self):
  134. return Lexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore)
  135. def _build_parser(self):
  136. self.parser_class = get_frontend(self.options.parser, self.options.lexer)
  137. self.parse_tree_builder = ParseTreeBuilder(self.rules, self.options.tree_class, self.options.propagate_positions, self.options.keep_all_tokens)
  138. rules, callback = self.parse_tree_builder.apply(self.options.transformer)
  139. if self.profiler:
  140. for f in dir(callback):
  141. if not (f.startswith('__') and f.endswith('__')):
  142. setattr(callback, f, self.profiler.make_wrapper('transformer', getattr(callback, f)))
  143. parser_conf = ParserConf(rules, callback, self.options.start)
  144. return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
  145. def lex(self, text):
  146. if not hasattr(self, 'lexer'):
  147. self.lexer = self._build_lexer()
  148. stream = self.lexer.lex(text)
  149. if self.options.postlex:
  150. return self.options.postlex.process(stream)
  151. else:
  152. return stream
  153. def parse(self, text):
  154. return self.parser.parse(text)
  155. # if self.profiler:
  156. # self.profiler.enter_section('lex')
  157. # l = list(self.lex(text))
  158. # self.profiler.enter_section('parse')
  159. # try:
  160. # return self.parser.parse(l)
  161. # finally:
  162. # self.profiler.enter_section('outside_lark')
  163. # else:
  164. # l = list(self.lex(text))
  165. # return self.parser.parse(l)