This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

207 lines
6.7 KiB

  1. ###{standalone
  2. #
  3. #
  4. # Lark Stand-alone Generator Tool
  5. # ----------------------------------
  6. # Generates a stand-alone LALR(1) parser with a standard lexer
  7. #
  8. # Git: https://github.com/erezsh/lark
  9. # Author: Erez Shinan (erezshin@gmail.com)
  10. #
  11. #
  12. # >>> LICENSE
  13. #
  14. # This tool and its generated code use a separate license from Lark.
  15. #
  16. # It is licensed under GPLv2 or above.
  17. #
  18. # If you wish to purchase a commercial license for this tool and its
  19. # generated code, contact me via email.
  20. #
  21. # This program is free software: you can redistribute it and/or modify
  22. # it under the terms of the GNU General Public License as published by
  23. # the Free Software Foundation, either version 2 of the License, or
  24. # (at your option) any later version.
  25. #
  26. # This program is distributed in the hope that it will be useful,
  27. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  28. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  29. # GNU General Public License for more details.
  30. #
  31. # See <http://www.gnu.org/licenses/>.
  32. #
  33. #
  34. ###}
  35. import codecs
  36. import sys
  37. import os
  38. from pprint import pprint
  39. from os import path
  40. from collections import defaultdict
  41. import lark
  42. from lark import Lark
  43. from lark.parsers.lalr_analysis import Shift, Reduce
  44. from ..grammar import Rule
  45. __dir__ = path.dirname(__file__)
  46. __larkdir__ = path.join(__dir__, path.pardir)
  47. EXTRACT_STANDALONE_FILES = [
  48. 'tools/standalone.py',
  49. 'utils.py',
  50. 'common.py',
  51. 'tree.py',
  52. 'indenter.py',
  53. 'lexer.py',
  54. 'parse_tree_builder.py',
  55. 'parsers/lalr_parser.py',
  56. ]
  57. def extract_sections(lines):
  58. section = None
  59. text = []
  60. sections = defaultdict(list)
  61. for l in lines:
  62. if l.startswith('###'):
  63. if l[3] == '{':
  64. section = l[4:].strip()
  65. elif l[3] == '}':
  66. sections[section] += text
  67. section = None
  68. text = []
  69. else:
  70. raise ValueError(l)
  71. elif section:
  72. text.append(l)
  73. return {name:''.join(text) for name, text in sections.items()}
  74. class LexerAtoms:
  75. def __init__(self, lexer):
  76. self.mres = [(p.pattern,d) for p,d in lexer.mres]
  77. self.newline_types = lexer.newline_types
  78. self.ignore_types = lexer.ignore_types
  79. self.callback = {name:[(p.pattern,d) for p,d in c.mres]
  80. for name, c in lexer.callback.items()}
  81. def print_python(self):
  82. print('import re')
  83. print('MRES = (')
  84. pprint(self.mres)
  85. print(')')
  86. print('LEXER_CALLBACK = (')
  87. pprint(self.callback)
  88. print(')')
  89. print('NEWLINE_TYPES = %s' % self.newline_types)
  90. print('IGNORE_TYPES = %s' % self.ignore_types)
  91. print('class LexerRegexps: pass')
  92. print('lexer_regexps = LexerRegexps()')
  93. print('lexer_regexps.mres = [(re.compile(p), d) for p, d in MRES]')
  94. print('lexer_regexps.callback = {n: UnlessCallback([(re.compile(p), d) for p, d in mres])')
  95. print(' for n, mres in LEXER_CALLBACK.items()}')
  96. print('lexer = _Lex(lexer_regexps)')
  97. print('def lex(stream):')
  98. print(' return lexer.lex(stream, NEWLINE_TYPES, IGNORE_TYPES)')
  99. class GetRule:
  100. def __init__(self, rule_id):
  101. self.rule_id = rule_id
  102. def __repr__(self):
  103. return 'RULES[%d]' % self.rule_id
  104. rule_ids = {}
  105. token_types = {}
  106. def _get_token_type(token_type):
  107. if token_type not in token_types:
  108. token_types[token_type] = len(token_types)
  109. return token_types[token_type]
  110. class ParserAtoms:
  111. def __init__(self, parser):
  112. self.parse_table = parser._parse_table
  113. def print_python(self):
  114. print('class ParseTable: pass')
  115. print('parse_table = ParseTable()')
  116. print('STATES = {')
  117. for state, actions in self.parse_table.states.items():
  118. print(' %r: %r,' % (state, {_get_token_type(token): ((1, rule_ids[arg]) if action is Reduce else (0, arg))
  119. for token, (action, arg) in actions.items()}))
  120. print('}')
  121. print('TOKEN_TYPES = (')
  122. pprint({v:k for k, v in token_types.items()})
  123. print(')')
  124. print('parse_table.states = {s: {TOKEN_TYPES[t]: (a, RULES[x] if a is Reduce else x) for t, (a, x) in acts.items()}')
  125. print(' for s, acts in STATES.items()}')
  126. print('parse_table.start_state = %s' % self.parse_table.start_state)
  127. print('parse_table.end_state = %s' % self.parse_table.end_state)
  128. print('class Lark_StandAlone:')
  129. print(' def __init__(self, transformer=None, postlex=None):')
  130. print(' callback = parse_tree_builder.create_callback(transformer=transformer)')
  131. print(' callbacks = {rule: getattr(callback, rule.alias or rule.origin, None) for rule in RULES.values()}')
  132. print(' self.parser = _Parser(parse_table, callbacks)')
  133. print(' self.postlex = postlex')
  134. print(' def parse(self, stream):')
  135. print(' tokens = lex(stream)')
  136. print(' if self.postlex: tokens = self.postlex.process(tokens)')
  137. print(' return self.parser.parse(tokens)')
  138. class TreeBuilderAtoms:
  139. def __init__(self, lark):
  140. self.rules = lark.rules
  141. self.ptb = lark._parse_tree_builder
  142. def print_python(self):
  143. print('RULES = {')
  144. for i, r in enumerate(self.rules):
  145. rule_ids[r] = i
  146. print(' %d: Rule(%r, %r, %r, %r),' % (i, r.origin, r.expansion, self.ptb.user_aliases[r], r.options ))
  147. print('}')
  148. print('parse_tree_builder = ParseTreeBuilder(RULES.values(), Tree)')
  149. def main(fobj, start):
  150. lark_inst = Lark(fobj, parser="lalr", start=start)
  151. lexer_atoms = LexerAtoms(lark_inst.parser.lexer)
  152. parser_atoms = ParserAtoms(lark_inst.parser.parser)
  153. tree_builder_atoms = TreeBuilderAtoms(lark_inst)
  154. print('# The file was automatically generated by Lark v%s' % lark.__version__)
  155. for pyfile in EXTRACT_STANDALONE_FILES:
  156. with open(os.path.join(__larkdir__, pyfile)) as f:
  157. print (extract_sections(f)['standalone'])
  158. with open(os.path.join(__larkdir__, 'grammar.py')) as grammar_py:
  159. print(grammar_py.read())
  160. print('Shift = 0')
  161. print('Reduce = 1')
  162. lexer_atoms.print_python()
  163. tree_builder_atoms.print_python()
  164. parser_atoms.print_python()
  165. if __name__ == '__main__':
  166. if len(sys.argv) < 2:
  167. print("Lark Stand-alone Generator Tool")
  168. print("Usage: python -m lark.tools.standalone <grammar-file> [<start>]")
  169. sys.exit(1)
  170. if len(sys.argv) == 3:
  171. fn, start = sys.argv[1:]
  172. elif len(sys.argv) == 2:
  173. fn, start = sys.argv[1], 'start'
  174. else:
  175. assert False, sys.argv
  176. with codecs.open(fn, encoding='utf8') as f:
  177. main(f, start)