This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

480 lines
18 KiB

  1. from __future__ import absolute_import
  2. import unittest
  3. import logging
  4. import os
  5. import sys
  6. try:
  7. from cStringIO import StringIO as cStringIO
  8. except ImportError:
  9. # Available only in Python 2.x, 3.x only has io.StringIO from below
  10. cStringIO = None
  11. from io import (
  12. StringIO as uStringIO,
  13. open,
  14. )
  15. logging.basicConfig(level=logging.INFO)
  16. from lark.lark import Lark
  17. from lark.common import GrammarError, ParseError
  18. from lark.lexer import LexError
  19. __path__ = os.path.dirname(__file__)
  20. def _read(n, *args):
  21. with open(os.path.join(__path__, n), *args) as f:
  22. return f.read()
  23. class TestParsers(unittest.TestCase):
  24. def test_same_ast(self):
  25. "Tests that Earley and LALR parsers produce equal trees"
  26. g = Lark("""start: "(" name_list ("," "*" NAME)? ")"
  27. name_list: NAME | name_list "," NAME
  28. NAME: /\w+/ """, parser='lalr')
  29. l = g.parse('(a,b,c,*x)')
  30. g = Lark("""start: "(" name_list ("," "*" NAME)? ")"
  31. name_list: NAME | name_list "," NAME
  32. NAME: /\w/+ """)
  33. l2 = g.parse('(a,b,c,*x)')
  34. assert l == l2, '%s != %s' % (l.pretty(), l2.pretty())
  35. class TestEarley(unittest.TestCase):
  36. def test_anon_in_scanless(self):
  37. # Fails an Earley implementation without special handling for empty rules,
  38. # or re-processing of already completed rules.
  39. g = Lark(r"""start: B
  40. B: ("ab"|/[^b]/)*
  41. """, lexer=None)
  42. assertEqual( g.parse('abc'), 'abc')
  43. def test_earley_scanless(self):
  44. g = Lark("""start: A "b" c
  45. A: "a"+
  46. c: "abc"
  47. """, parser="earley", lexer=None)
  48. x = g.parse('aaaababc')
  49. def _make_parser_test(LEXER, PARSER):
  50. def _Lark(grammar, **kwargs):
  51. return Lark(grammar, lexer=LEXER, parser=PARSER, **kwargs)
  52. class _TestParser(unittest.TestCase):
  53. def test_basic1(self):
  54. g = _Lark("""start: a+ b a* "b" a*
  55. b: "b"
  56. a: "a"
  57. """)
  58. r = g.parse('aaabaab')
  59. self.assertEqual( ''.join(x.data for x in r.children), 'aaabaa' )
  60. r = g.parse('aaabaaba')
  61. self.assertEqual( ''.join(x.data for x in r.children), 'aaabaaa' )
  62. self.assertRaises(ParseError, g.parse, 'aaabaa')
  63. def test_basic2(self):
  64. # Multiple parsers and colliding tokens
  65. g = _Lark("""start: B A
  66. B: "12"
  67. A: "1" """)
  68. g2 = _Lark("""start: B A
  69. B: "12"
  70. A: "2" """)
  71. x = g.parse('121')
  72. assert x.data == 'start' and x.children == ['12', '1'], x
  73. x = g2.parse('122')
  74. assert x.data == 'start' and x.children == ['12', '2'], x
  75. @unittest.skipIf(cStringIO is None, "cStringIO not available")
  76. def test_stringio_bytes(self):
  77. """Verify that a Lark can be created from file-like objects other than Python's standard 'file' object"""
  78. _Lark(cStringIO(b'start: a+ b a* "b" a*\n b: "b"\n a: "a" '))
  79. def test_stringio_unicode(self):
  80. """Verify that a Lark can be created from file-like objects other than Python's standard 'file' object"""
  81. _Lark(uStringIO(u'start: a+ b a* "b" a*\n b: "b"\n a: "a" '))
  82. def test_unicode(self):
  83. g = _Lark(u"""start: UNIA UNIB UNIA
  84. UNIA: /\xa3/
  85. UNIB: /\u0101/
  86. """)
  87. g.parse(u'\xa3\u0101\u00a3')
  88. @unittest.skipIf(LEXER is None, "Regexps >1 not supported with scanless parsing")
  89. def test_unicode2(self):
  90. g = _Lark(r"""start: UNIA UNIB UNIA UNIC
  91. UNIA: /\xa3/
  92. UNIB: "a\u0101b\ "
  93. UNIC: /a?\u0101c\n/
  94. """)
  95. g.parse(u'\xa3a\u0101b\\ \u00a3\u0101c\n')
  96. def test_unicode3(self):
  97. g = _Lark(r"""start: UNIA UNIB UNIA UNIC
  98. UNIA: /\xa3/
  99. UNIB: "\u0101"
  100. UNIC: /\u0203/ /\n/
  101. """)
  102. g.parse(u'\xa3\u0101\u00a3\u0203\n')
  103. def test_stack_for_ebnf(self):
  104. """Verify that stack depth isn't an issue for EBNF grammars"""
  105. g = _Lark(r"""start: a+
  106. a : "a" """)
  107. g.parse("a" * (sys.getrecursionlimit()*2 ))
  108. def test_expand1_lists_with_one_item(self):
  109. g = _Lark(r"""start: list
  110. ?list: item+
  111. item : A
  112. A: "a"
  113. """)
  114. r = g.parse("a")
  115. # because 'list' is an expand-if-contains-one rule and we only provided one element it should have expanded to 'item'
  116. self.assertSequenceEqual([subtree.data for subtree in r.children], ('item',))
  117. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  118. self.assertEqual(len(r.children), 1)
  119. def test_expand1_lists_with_one_item_2(self):
  120. g = _Lark(r"""start: list
  121. ?list: item+ "!"
  122. item : A
  123. A: "a"
  124. """)
  125. r = g.parse("a!")
  126. # because 'list' is an expand-if-contains-one rule and we only provided one element it should have expanded to 'item'
  127. self.assertSequenceEqual([subtree.data for subtree in r.children], ('item',))
  128. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  129. self.assertEqual(len(r.children), 1)
  130. def test_dont_expand1_lists_with_multiple_items(self):
  131. g = _Lark(r"""start: list
  132. ?list: item+
  133. item : A
  134. A: "a"
  135. """)
  136. r = g.parse("aa")
  137. # because 'list' is an expand-if-contains-one rule and we've provided more than one element it should *not* have expanded
  138. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  139. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  140. self.assertEqual(len(r.children), 1)
  141. # Sanity check: verify that 'list' contains the two 'item's we've given it
  142. [list] = r.children
  143. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  144. def test_dont_expand1_lists_with_multiple_items_2(self):
  145. g = _Lark(r"""start: list
  146. ?list: item+ "!"
  147. item : A
  148. A: "a"
  149. """)
  150. r = g.parse("aa!")
  151. # because 'list' is an expand-if-contains-one rule and we've provided more than one element it should *not* have expanded
  152. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  153. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  154. self.assertEqual(len(r.children), 1)
  155. # Sanity check: verify that 'list' contains the two 'item's we've given it
  156. [list] = r.children
  157. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  158. def test_empty_expand1_list(self):
  159. g = _Lark(r"""start: list
  160. ?list: item*
  161. item : A
  162. A: "a"
  163. """)
  164. r = g.parse("")
  165. # because 'list' is an expand-if-contains-one rule and we've provided less than one element (i.e. none) it should *not* have expanded
  166. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  167. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  168. self.assertEqual(len(r.children), 1)
  169. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  170. [list] = r.children
  171. self.assertSequenceEqual([item.data for item in list.children], ())
  172. def test_empty_expand1_list_2(self):
  173. g = _Lark(r"""start: list
  174. ?list: item* "!"?
  175. item : A
  176. A: "a"
  177. """)
  178. r = g.parse("")
  179. # because 'list' is an expand-if-contains-one rule and we've provided less than one element (i.e. none) it should *not* have expanded
  180. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  181. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  182. self.assertEqual(len(r.children), 1)
  183. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  184. [list] = r.children
  185. self.assertSequenceEqual([item.data for item in list.children], ())
  186. def test_empty_flatten_list(self):
  187. g = _Lark(r"""start: list
  188. list: | item "," list
  189. item : A
  190. A: "a"
  191. """)
  192. r = g.parse("")
  193. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  194. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  195. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  196. [list] = r.children
  197. self.assertSequenceEqual([item.data for item in list.children], ())
  198. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  199. def test_single_item_flatten_list(self):
  200. g = _Lark(r"""start: list
  201. list: | item "," list
  202. item : A
  203. A: "a"
  204. """)
  205. r = g.parse("a,")
  206. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  207. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  208. # Sanity check: verify that 'list' contains exactly the one 'item' we've given it
  209. [list] = r.children
  210. self.assertSequenceEqual([item.data for item in list.children], ('item',))
  211. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  212. def test_multiple_item_flatten_list(self):
  213. g = _Lark(r"""start: list
  214. #list: | item "," list
  215. item : A
  216. A: "a"
  217. """)
  218. r = g.parse("a,a,")
  219. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  220. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  221. # Sanity check: verify that 'list' contains exactly the two 'item's we've given it
  222. [list] = r.children
  223. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  224. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  225. def test_recurse_flatten(self):
  226. """Verify that stack depth doesn't get exceeded on recursive rules marked for flattening."""
  227. g = _Lark(r"""start: a | start a
  228. a : A
  229. A : "a" """)
  230. # Force PLY to write to the debug log, but prevent writing it to the terminal (uses repr() on the half-built
  231. # STree data structures, which uses recursion).
  232. g.parse("a" * (sys.getrecursionlimit() // 4))
  233. def test_token_collision(self):
  234. g = _Lark("""start: "Hello" NAME
  235. NAME: /\w/+
  236. %ignore " "
  237. """)
  238. x = g.parse('Hello World')
  239. self.assertSequenceEqual(x.children, ['World'])
  240. x = g.parse('Hello HelloWorld')
  241. self.assertSequenceEqual(x.children, ['HelloWorld'])
  242. # def test_string_priority(self):
  243. # g = _Lark("""start: (A | /a?bb/)+
  244. # A: "a" """)
  245. # x = g.parse('abb')
  246. # self.assertEqual(len(x.children), 2)
  247. # # This parse raises an exception because the lexer will always try to consume
  248. # # "a" first and will never match the regular expression
  249. # # This behavior is subject to change!!
  250. # # Thie won't happen with ambiguity handling.
  251. # g = _Lark("""start: (A | /a?ab/)+
  252. # A: "a" """)
  253. # self.assertRaises(LexError, g.parse, 'aab')
  254. def test_undefined_rule(self):
  255. self.assertRaises(GrammarError, _Lark, """start: a""")
  256. def test_undefined_token(self):
  257. self.assertRaises(GrammarError, _Lark, """start: A""")
  258. def test_rule_collision(self):
  259. g = _Lark("""start: "a"+ "b"
  260. | "a"+ """)
  261. x = g.parse('aaaa')
  262. x = g.parse('aaaab')
  263. def test_rule_collision2(self):
  264. g = _Lark("""start: "a"* "b"
  265. | "a"+ """)
  266. x = g.parse('aaaa')
  267. x = g.parse('aaaab')
  268. x = g.parse('b')
  269. @unittest.skipIf(LEXER is None, "Regexps >1 not supported with scanless parsing")
  270. def test_regex_embed(self):
  271. g = _Lark("""start: A B C
  272. A: /a/
  273. B: /${A}b/
  274. C: /${B}c/
  275. """)
  276. x = g.parse('aababc')
  277. def test_token_embed(self):
  278. g = _Lark("""start: A B C
  279. A: "a"
  280. B: A "b"
  281. C: B "c"
  282. """)
  283. x = g.parse('aababc')
  284. def test_token_not_anon(self):
  285. """Tests that "a" is matched as A, rather than an anonymous token.
  286. That means that "a" is not filtered out, despite being an 'immediate string'.
  287. Whether or not this is the intuitive behavior, I'm not sure yet.
  288. Perhaps the right thing to do is report a collision (if such is relevant)
  289. -Erez
  290. """
  291. g = _Lark("""start: "a"
  292. A: "a" """)
  293. x = g.parse('a')
  294. self.assertEqual(len(x.children), 1, '"a" should not be considered anonymous')
  295. self.assertEqual(x.children[0].type, "A")
  296. g = _Lark("""start: /a/
  297. A: /a/ """)
  298. x = g.parse('a')
  299. self.assertEqual(len(x.children), 1, '/a/ should not be considered anonymous')
  300. self.assertEqual(x.children[0].type, "A")
  301. def test_maybe(self):
  302. g = _Lark("""start: ["a"] """)
  303. x = g.parse('a')
  304. x = g.parse('')
  305. def test_start(self):
  306. g = _Lark("""a: "a" a? """, start='a')
  307. x = g.parse('a')
  308. x = g.parse('aa')
  309. x = g.parse('aaa')
  310. def test_alias(self):
  311. g = _Lark("""start: "a" -> b """)
  312. x = g.parse('a')
  313. self.assertEqual(x.data, "b")
  314. def test_token_ebnf(self):
  315. g = _Lark("""start: A
  316. A: "a"* ("b"? "c".."e")+
  317. """)
  318. x = g.parse('abcde')
  319. x = g.parse('dd')
  320. def test_backslash(self):
  321. g = _Lark(r"""start: "\\" "a"
  322. """)
  323. x = g.parse(r'\a')
  324. g = _Lark(r"""start: /\\\\/ /a/
  325. """)
  326. x = g.parse(r'\a')
  327. def test_special_chars(self):
  328. g = _Lark(r"""start: "\n"
  329. """)
  330. x = g.parse('\n')
  331. g = _Lark(r"""start: /\n/
  332. """)
  333. x = g.parse('\n')
  334. def test_backslash2(self):
  335. g = _Lark(r"""start: "\"" "-"
  336. """)
  337. x = g.parse('"-')
  338. g = _Lark(r"""start: /\// /-/
  339. """)
  340. x = g.parse('/-')
  341. # def test_token_recurse(self):
  342. # g = _Lark("""start: A
  343. # A: B
  344. # B: A
  345. # """)
  346. def test_empty(self):
  347. # Fails an Earley implementation without special handling for empty rules,
  348. # or re-processing of already completed rules.
  349. g = _Lark(r"""start: _empty a "B"
  350. a: _empty "A"
  351. _empty:
  352. """)
  353. x = g.parse('AB')
  354. def test_lexer_token_limit(self):
  355. "Python has a stupid limit of 100 groups in a regular expression. Test that we handle this limitation"
  356. tokens = {'A%d'%i:'"%d"'%i for i in range(300)}
  357. g = _Lark("""start: %s
  358. %s""" % (' '.join(tokens), '\n'.join("%s: %s"%x for x in tokens.items())))
  359. def test_float_without_lexer(self):
  360. g = _Lark("""start: ["+"|"-"] float
  361. float: digit* "." digit+ exp?
  362. | digit+ exp
  363. exp: ("e"|"E") ["+"|"-"] digit+
  364. digit: "0"|"1"|"2"|"3"|"4"|"5"|"6"|"7"|"8"|"9"
  365. """)
  366. g.parse("1.2")
  367. g.parse("-.2e9")
  368. g.parse("+2e-9")
  369. self.assertRaises(ParseError, g.parse, "+2e-9e")
  370. _NAME = "Test" + PARSER.capitalize() + (LEXER or 'Scanless').capitalize()
  371. _TestParser.__name__ = _NAME
  372. globals()[_NAME] = _TestParser
  373. _TO_TEST = [
  374. ('standard', 'earley'),
  375. ('standard', 'lalr'),
  376. ('contextual', 'lalr'),
  377. (None, 'earley'),
  378. ]
  379. for LEXER, PARSER in _TO_TEST:
  380. _make_parser_test(LEXER, PARSER)
  381. if __name__ == '__main__':
  382. unittest.main()