This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1373 lines
46 KiB

  1. # -*- coding: utf-8 -*-
  2. from __future__ import absolute_import
  3. import unittest
  4. import logging
  5. import os
  6. import sys
  7. try:
  8. from cStringIO import StringIO as cStringIO
  9. except ImportError:
  10. # Available only in Python 2.x, 3.x only has io.StringIO from below
  11. cStringIO = None
  12. from io import (
  13. StringIO as uStringIO,
  14. open,
  15. )
  16. logging.basicConfig(level=logging.INFO)
  17. from lark.lark import Lark
  18. from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput
  19. from lark.tree import Tree
  20. from lark.visitors import Transformer
  21. __path__ = os.path.dirname(__file__)
  22. def _read(n, *args):
  23. with open(os.path.join(__path__, n), *args) as f:
  24. return f.read()
  25. class TestParsers(unittest.TestCase):
  26. def test_same_ast(self):
  27. "Tests that Earley and LALR parsers produce equal trees"
  28. g = Lark(r"""start: "(" name_list ("," "*" NAME)? ")"
  29. name_list: NAME | name_list "," NAME
  30. NAME: /\w+/ """, parser='lalr')
  31. l = g.parse('(a,b,c,*x)')
  32. g = Lark(r"""start: "(" name_list ("," "*" NAME)? ")"
  33. name_list: NAME | name_list "," NAME
  34. NAME: /\w/+ """)
  35. l2 = g.parse('(a,b,c,*x)')
  36. assert l == l2, '%s != %s' % (l.pretty(), l2.pretty())
  37. def test_infinite_recurse(self):
  38. g = """start: a
  39. a: a | "a"
  40. """
  41. self.assertRaises(GrammarError, Lark, g, parser='lalr')
  42. # TODO: should it? shouldn't it?
  43. # l = Lark(g, parser='earley', lexer='dynamic')
  44. # self.assertRaises(ParseError, l.parse, 'a')
  45. def test_propagate_positions(self):
  46. g = Lark("""start: a
  47. a: "a"
  48. """, propagate_positions=True)
  49. r = g.parse('a')
  50. self.assertEqual( r.children[0].meta.line, 1 )
  51. def test_expand1(self):
  52. g = Lark("""start: a
  53. ?a: b
  54. b: "x"
  55. """)
  56. r = g.parse('x')
  57. self.assertEqual( r.children[0].data, "b" )
  58. g = Lark("""start: a
  59. ?a: b -> c
  60. b: "x"
  61. """)
  62. r = g.parse('x')
  63. self.assertEqual( r.children[0].data, "c" )
  64. g = Lark("""start: a
  65. ?a: B -> c
  66. B: "x"
  67. """)
  68. self.assertEqual( r.children[0].data, "c" )
  69. g = Lark("""start: a
  70. ?a: b b -> c
  71. b: "x"
  72. """)
  73. r = g.parse('xx')
  74. self.assertEqual( r.children[0].data, "c" )
  75. def test_embedded_transformer(self):
  76. class T(Transformer):
  77. def a(self, children):
  78. return "<a>"
  79. def b(self, children):
  80. return "<b>"
  81. def c(self, children):
  82. return "<c>"
  83. # Test regular
  84. g = Lark("""start: a
  85. a : "x"
  86. """, parser='lalr')
  87. r = T().transform(g.parse("x"))
  88. self.assertEqual( r.children, ["<a>"] )
  89. g = Lark("""start: a
  90. a : "x"
  91. """, parser='lalr', transformer=T())
  92. r = g.parse("x")
  93. self.assertEqual( r.children, ["<a>"] )
  94. # Test Expand1
  95. g = Lark("""start: a
  96. ?a : b
  97. b : "x"
  98. """, parser='lalr')
  99. r = T().transform(g.parse("x"))
  100. self.assertEqual( r.children, ["<b>"] )
  101. g = Lark("""start: a
  102. ?a : b
  103. b : "x"
  104. """, parser='lalr', transformer=T())
  105. r = g.parse("x")
  106. self.assertEqual( r.children, ["<b>"] )
  107. # Test Expand1 -> Alias
  108. g = Lark("""start: a
  109. ?a : b b -> c
  110. b : "x"
  111. """, parser='lalr')
  112. r = T().transform(g.parse("xx"))
  113. self.assertEqual( r.children, ["<c>"] )
  114. g = Lark("""start: a
  115. ?a : b b -> c
  116. b : "x"
  117. """, parser='lalr', transformer=T())
  118. r = g.parse("xx")
  119. self.assertEqual( r.children, ["<c>"] )
  120. def test_alias(self):
  121. Lark("""start: ["a"] "b" ["c"] "e" ["f"] ["g"] ["h"] "x" -> d """)
  122. def _make_full_earley_test(LEXER):
  123. def _Lark(grammar, **kwargs):
  124. return Lark(grammar, lexer=LEXER, parser='earley', propagate_positions=True, **kwargs)
  125. class _TestFullEarley(unittest.TestCase):
  126. def test_anon(self):
  127. # Fails an Earley implementation without special handling for empty rules,
  128. # or re-processing of already completed rules.
  129. g = Lark(r"""start: B
  130. B: ("ab"|/[^b]/)+
  131. """, lexer=LEXER)
  132. self.assertEqual( g.parse('abc').children[0], 'abc')
  133. def test_earley(self):
  134. g = Lark("""start: A "b" c
  135. A: "a"+
  136. c: "abc"
  137. """, parser="earley", lexer=LEXER)
  138. x = g.parse('aaaababc')
  139. def test_earley2(self):
  140. grammar = """
  141. start: statement+
  142. statement: "r"
  143. | "c" /[a-z]/+
  144. %ignore " "
  145. """
  146. program = """c b r"""
  147. l = Lark(grammar, parser='earley', lexer=LEXER)
  148. l.parse(program)
  149. @unittest.skipIf(LEXER=='dynamic', "Only relevant for the dynamic_complete parser")
  150. def test_earley3(self):
  151. """Tests prioritization and disambiguation for pseudo-terminals (there should be only one result)
  152. By default, `+` should immitate regexp greedy-matching
  153. """
  154. grammar = """
  155. start: A A
  156. A: "a"+
  157. """
  158. l = Lark(grammar, parser='earley', lexer=LEXER)
  159. res = l.parse("aaa")
  160. self.assertEqual(set(res.children), {'aa', 'a'})
  161. # XXX TODO fix Earley to maintain correct order
  162. # i.e. terminals it imitate greedy search for terminals, but lazy search for rules
  163. # self.assertEqual(res.children, ['aa', 'a'])
  164. def test_earley4(self):
  165. grammar = """
  166. start: A A?
  167. A: "a"+
  168. """
  169. l = Lark(grammar, parser='earley', lexer=LEXER)
  170. res = l.parse("aaa")
  171. assert set(res.children) == {'aa', 'a'} or res.children == ['aaa']
  172. # XXX TODO fix Earley to maintain correct order
  173. # i.e. terminals it imitate greedy search for terminals, but lazy search for rules
  174. # self.assertEqual(res.children, ['aaa'])
  175. def test_earley_repeating_empty(self):
  176. # This was a sneaky bug!
  177. grammar = """
  178. !start: "a" empty empty "b"
  179. empty: empty2
  180. empty2:
  181. """
  182. parser = Lark(grammar, parser='earley', lexer=LEXER)
  183. res = parser.parse('ab')
  184. empty_tree = Tree('empty', [Tree('empty2', [])])
  185. self.assertSequenceEqual(res.children, ['a', empty_tree, empty_tree, 'b'])
  186. @unittest.skipIf(LEXER=='standard', "Requires dynamic lexer")
  187. def test_earley_explicit_ambiguity(self):
  188. # This was a sneaky bug!
  189. grammar = """
  190. start: a b | ab
  191. a: "a"
  192. b: "b"
  193. ab: "ab"
  194. """
  195. parser = Lark(grammar, parser='earley', lexer=LEXER, ambiguity='explicit')
  196. ambig_tree = parser.parse('ab')
  197. self.assertEqual( ambig_tree.data, '_ambig')
  198. self.assertEqual( len(ambig_tree.children), 2)
  199. @unittest.skipIf(LEXER=='standard', "Requires dynamic lexer")
  200. def test_ambiguity1(self):
  201. grammar = """
  202. start: cd+ "e"
  203. !cd: "c"
  204. | "d"
  205. | "cd"
  206. """
  207. l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER)
  208. ambig_tree = l.parse('cde')
  209. assert ambig_tree.data == '_ambig', ambig_tree
  210. assert len(ambig_tree.children) == 2
  211. @unittest.skipIf(LEXER=='standard', "Requires dynamic lexer")
  212. def test_ambiguity2(self):
  213. grammar = """
  214. ANY: /[a-zA-Z0-9 ]+/
  215. a.2: "A" b+
  216. b.2: "B"
  217. c: ANY
  218. start: (a|c)*
  219. """
  220. l = Lark(grammar, parser='earley', lexer=LEXER)
  221. res = l.parse('ABX')
  222. expected = Tree('start', [
  223. Tree('a', [
  224. Tree('b', [])
  225. ]),
  226. Tree('c', [
  227. 'X'
  228. ])
  229. ])
  230. self.assertEqual(res, expected)
  231. def test_fruitflies_ambig(self):
  232. grammar = """
  233. start: noun verb noun -> simple
  234. | noun verb "like" noun -> comparative
  235. noun: adj? NOUN
  236. verb: VERB
  237. adj: ADJ
  238. NOUN: "flies" | "bananas" | "fruit"
  239. VERB: "like" | "flies"
  240. ADJ: "fruit"
  241. %import common.WS
  242. %ignore WS
  243. """
  244. parser = Lark(grammar, ambiguity='explicit', lexer=LEXER)
  245. tree = parser.parse('fruit flies like bananas')
  246. expected = Tree('_ambig', [
  247. Tree('comparative', [
  248. Tree('noun', ['fruit']),
  249. Tree('verb', ['flies']),
  250. Tree('noun', ['bananas'])
  251. ]),
  252. Tree('simple', [
  253. Tree('noun', [Tree('adj', ['fruit']), 'flies']),
  254. Tree('verb', ['like']),
  255. Tree('noun', ['bananas'])
  256. ])
  257. ])
  258. # self.assertEqual(tree, expected)
  259. self.assertEqual(tree.data, expected.data)
  260. self.assertEqual(set(tree.children), set(expected.children))
  261. @unittest.skipIf(LEXER!='dynamic_complete', "Only relevant for the dynamic_complete parser")
  262. def test_explicit_ambiguity2(self):
  263. grammar = r"""
  264. start: NAME+
  265. NAME: /\w+/
  266. %ignore " "
  267. """
  268. text = """cat"""
  269. parser = _Lark(grammar, start='start', ambiguity='explicit')
  270. tree = parser.parse(text)
  271. self.assertEqual(tree.data, '_ambig')
  272. combinations = {tuple(str(s) for s in t.children) for t in tree.children}
  273. self.assertEqual(combinations, {
  274. ('cat',),
  275. ('ca', 't'),
  276. ('c', 'at'),
  277. ('c', 'a' ,'t')
  278. })
  279. def test_term_ambig_resolve(self):
  280. grammar = r"""
  281. !start: NAME+
  282. NAME: /\w+/
  283. %ignore " "
  284. """
  285. text = """foo bar"""
  286. parser = Lark(grammar)
  287. tree = parser.parse(text)
  288. self.assertEqual(tree.children, ['foo', 'bar'])
  289. # @unittest.skipIf(LEXER=='dynamic', "Not implemented in Dynamic Earley yet") # TODO
  290. # def test_not_all_derivations(self):
  291. # grammar = """
  292. # start: cd+ "e"
  293. # !cd: "c"
  294. # | "d"
  295. # | "cd"
  296. # """
  297. # l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER, earley__all_derivations=False)
  298. # x = l.parse('cde')
  299. # assert x.data != '_ambig', x
  300. # assert len(x.children) == 1
  301. _NAME = "TestFullEarley" + LEXER.capitalize()
  302. _TestFullEarley.__name__ = _NAME
  303. globals()[_NAME] = _TestFullEarley
  304. def _make_parser_test(LEXER, PARSER):
  305. def _Lark(grammar, **kwargs):
  306. return Lark(grammar, lexer=LEXER, parser=PARSER, propagate_positions=True, **kwargs)
  307. class _TestParser(unittest.TestCase):
  308. def test_basic1(self):
  309. g = _Lark("""start: a+ b a* "b" a*
  310. b: "b"
  311. a: "a"
  312. """)
  313. r = g.parse('aaabaab')
  314. self.assertEqual( ''.join(x.data for x in r.children), 'aaabaa' )
  315. r = g.parse('aaabaaba')
  316. self.assertEqual( ''.join(x.data for x in r.children), 'aaabaaa' )
  317. self.assertRaises(ParseError, g.parse, 'aaabaa')
  318. def test_basic2(self):
  319. # Multiple parsers and colliding tokens
  320. g = _Lark("""start: B A
  321. B: "12"
  322. A: "1" """)
  323. g2 = _Lark("""start: B A
  324. B: "12"
  325. A: "2" """)
  326. x = g.parse('121')
  327. assert x.data == 'start' and x.children == ['12', '1'], x
  328. x = g2.parse('122')
  329. assert x.data == 'start' and x.children == ['12', '2'], x
  330. @unittest.skipIf(cStringIO is None, "cStringIO not available")
  331. def test_stringio_bytes(self):
  332. """Verify that a Lark can be created from file-like objects other than Python's standard 'file' object"""
  333. _Lark(cStringIO(b'start: a+ b a* "b" a*\n b: "b"\n a: "a" '))
  334. def test_stringio_unicode(self):
  335. """Verify that a Lark can be created from file-like objects other than Python's standard 'file' object"""
  336. _Lark(uStringIO(u'start: a+ b a* "b" a*\n b: "b"\n a: "a" '))
  337. def test_unicode(self):
  338. g = _Lark(u"""start: UNIA UNIB UNIA
  339. UNIA: /\xa3/
  340. UNIB: /\u0101/
  341. """)
  342. g.parse(u'\xa3\u0101\u00a3')
  343. def test_unicode2(self):
  344. g = _Lark(r"""start: UNIA UNIB UNIA UNIC
  345. UNIA: /\xa3/
  346. UNIB: "a\u0101b\ "
  347. UNIC: /a?\u0101c\n/
  348. """)
  349. g.parse(u'\xa3a\u0101b\\ \u00a3\u0101c\n')
  350. def test_unicode3(self):
  351. g = _Lark(r"""start: UNIA UNIB UNIA UNIC
  352. UNIA: /\xa3/
  353. UNIB: "\u0101"
  354. UNIC: /\u0203/ /\n/
  355. """)
  356. g.parse(u'\xa3\u0101\u00a3\u0203\n')
  357. def test_hex_escape(self):
  358. g = _Lark(r"""start: A B C
  359. A: "\x01"
  360. B: /\x02/
  361. C: "\xABCD"
  362. """)
  363. g.parse('\x01\x02\xABCD')
  364. @unittest.skipIf(PARSER == 'cyk', "Takes forever")
  365. def test_stack_for_ebnf(self):
  366. """Verify that stack depth isn't an issue for EBNF grammars"""
  367. g = _Lark(r"""start: a+
  368. a : "a" """)
  369. g.parse("a" * (sys.getrecursionlimit()*2 ))
  370. def test_expand1_lists_with_one_item(self):
  371. g = _Lark(r"""start: list
  372. ?list: item+
  373. item : A
  374. A: "a"
  375. """)
  376. r = g.parse("a")
  377. # because 'list' is an expand-if-contains-one rule and we only provided one element it should have expanded to 'item'
  378. self.assertSequenceEqual([subtree.data for subtree in r.children], ('item',))
  379. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  380. self.assertEqual(len(r.children), 1)
  381. def test_expand1_lists_with_one_item_2(self):
  382. g = _Lark(r"""start: list
  383. ?list: item+ "!"
  384. item : A
  385. A: "a"
  386. """)
  387. r = g.parse("a!")
  388. # because 'list' is an expand-if-contains-one rule and we only provided one element it should have expanded to 'item'
  389. self.assertSequenceEqual([subtree.data for subtree in r.children], ('item',))
  390. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  391. self.assertEqual(len(r.children), 1)
  392. def test_dont_expand1_lists_with_multiple_items(self):
  393. g = _Lark(r"""start: list
  394. ?list: item+
  395. item : A
  396. A: "a"
  397. """)
  398. r = g.parse("aa")
  399. # because 'list' is an expand-if-contains-one rule and we've provided more than one element it should *not* have expanded
  400. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  401. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  402. self.assertEqual(len(r.children), 1)
  403. # Sanity check: verify that 'list' contains the two 'item's we've given it
  404. [list] = r.children
  405. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  406. def test_dont_expand1_lists_with_multiple_items_2(self):
  407. g = _Lark(r"""start: list
  408. ?list: item+ "!"
  409. item : A
  410. A: "a"
  411. """)
  412. r = g.parse("aa!")
  413. # because 'list' is an expand-if-contains-one rule and we've provided more than one element it should *not* have expanded
  414. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  415. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  416. self.assertEqual(len(r.children), 1)
  417. # Sanity check: verify that 'list' contains the two 'item's we've given it
  418. [list] = r.children
  419. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  420. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  421. def test_empty_expand1_list(self):
  422. g = _Lark(r"""start: list
  423. ?list: item*
  424. item : A
  425. A: "a"
  426. """)
  427. r = g.parse("")
  428. # because 'list' is an expand-if-contains-one rule and we've provided less than one element (i.e. none) it should *not* have expanded
  429. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  430. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  431. self.assertEqual(len(r.children), 1)
  432. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  433. [list] = r.children
  434. self.assertSequenceEqual([item.data for item in list.children], ())
  435. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  436. def test_empty_expand1_list_2(self):
  437. g = _Lark(r"""start: list
  438. ?list: item* "!"?
  439. item : A
  440. A: "a"
  441. """)
  442. r = g.parse("")
  443. # because 'list' is an expand-if-contains-one rule and we've provided less than one element (i.e. none) it should *not* have expanded
  444. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  445. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  446. self.assertEqual(len(r.children), 1)
  447. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  448. [list] = r.children
  449. self.assertSequenceEqual([item.data for item in list.children], ())
  450. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  451. def test_empty_flatten_list(self):
  452. g = _Lark(r"""start: list
  453. list: | item "," list
  454. item : A
  455. A: "a"
  456. """)
  457. r = g.parse("")
  458. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  459. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  460. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  461. [list] = r.children
  462. self.assertSequenceEqual([item.data for item in list.children], ())
  463. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  464. def test_single_item_flatten_list(self):
  465. g = _Lark(r"""start: list
  466. list: | item "," list
  467. item : A
  468. A: "a"
  469. """)
  470. r = g.parse("a,")
  471. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  472. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  473. # Sanity check: verify that 'list' contains exactly the one 'item' we've given it
  474. [list] = r.children
  475. self.assertSequenceEqual([item.data for item in list.children], ('item',))
  476. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  477. def test_multiple_item_flatten_list(self):
  478. g = _Lark(r"""start: list
  479. #list: | item "," list
  480. item : A
  481. A: "a"
  482. """)
  483. r = g.parse("a,a,")
  484. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  485. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  486. # Sanity check: verify that 'list' contains exactly the two 'item's we've given it
  487. [list] = r.children
  488. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  489. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  490. def test_recurse_flatten(self):
  491. """Verify that stack depth doesn't get exceeded on recursive rules marked for flattening."""
  492. g = _Lark(r"""start: a | start a
  493. a : A
  494. A : "a" """)
  495. # Force PLY to write to the debug log, but prevent writing it to the terminal (uses repr() on the half-built
  496. # STree data structures, which uses recursion).
  497. g.parse("a" * (sys.getrecursionlimit() // 4))
  498. def test_token_collision(self):
  499. g = _Lark(r"""start: "Hello" NAME
  500. NAME: /\w/+
  501. %ignore " "
  502. """)
  503. x = g.parse('Hello World')
  504. self.assertSequenceEqual(x.children, ['World'])
  505. x = g.parse('Hello HelloWorld')
  506. self.assertSequenceEqual(x.children, ['HelloWorld'])
  507. def test_token_collision_WS(self):
  508. g = _Lark(r"""start: "Hello" NAME
  509. NAME: /\w/+
  510. %import common.WS
  511. %ignore WS
  512. """)
  513. x = g.parse('Hello World')
  514. self.assertSequenceEqual(x.children, ['World'])
  515. x = g.parse('Hello HelloWorld')
  516. self.assertSequenceEqual(x.children, ['HelloWorld'])
  517. def test_token_collision2(self):
  518. g = _Lark("""
  519. !start: "starts"
  520. %import common.LCASE_LETTER
  521. """)
  522. x = g.parse("starts")
  523. self.assertSequenceEqual(x.children, ['starts'])
  524. # def test_string_priority(self):
  525. # g = _Lark("""start: (A | /a?bb/)+
  526. # A: "a" """)
  527. # x = g.parse('abb')
  528. # self.assertEqual(len(x.children), 2)
  529. # # This parse raises an exception because the lexer will always try to consume
  530. # # "a" first and will never match the regular expression
  531. # # This behavior is subject to change!!
  532. # # Thie won't happen with ambiguity handling.
  533. # g = _Lark("""start: (A | /a?ab/)+
  534. # A: "a" """)
  535. # self.assertRaises(LexError, g.parse, 'aab')
  536. def test_undefined_rule(self):
  537. self.assertRaises(GrammarError, _Lark, """start: a""")
  538. def test_undefined_token(self):
  539. self.assertRaises(GrammarError, _Lark, """start: A""")
  540. def test_rule_collision(self):
  541. g = _Lark("""start: "a"+ "b"
  542. | "a"+ """)
  543. x = g.parse('aaaa')
  544. x = g.parse('aaaab')
  545. def test_rule_collision2(self):
  546. g = _Lark("""start: "a"* "b"
  547. | "a"+ """)
  548. x = g.parse('aaaa')
  549. x = g.parse('aaaab')
  550. x = g.parse('b')
  551. def test_token_not_anon(self):
  552. """Tests that "a" is matched as an anonymous token, and not A.
  553. """
  554. g = _Lark("""start: "a"
  555. A: "a" """)
  556. x = g.parse('a')
  557. self.assertEqual(len(x.children), 0, '"a" should be considered anonymous')
  558. g = _Lark("""start: "a" A
  559. A: "a" """)
  560. x = g.parse('aa')
  561. self.assertEqual(len(x.children), 1, 'only "a" should be considered anonymous')
  562. self.assertEqual(x.children[0].type, "A")
  563. g = _Lark("""start: /a/
  564. A: /a/ """)
  565. x = g.parse('a')
  566. self.assertEqual(len(x.children), 1)
  567. self.assertEqual(x.children[0].type, "A", "A isn't associated with /a/")
  568. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  569. def test_maybe(self):
  570. g = _Lark("""start: ["a"] """)
  571. x = g.parse('a')
  572. x = g.parse('')
  573. def test_start(self):
  574. g = _Lark("""a: "a" a? """, start='a')
  575. x = g.parse('a')
  576. x = g.parse('aa')
  577. x = g.parse('aaa')
  578. def test_alias(self):
  579. g = _Lark("""start: "a" -> b """)
  580. x = g.parse('a')
  581. self.assertEqual(x.data, "b")
  582. def test_token_ebnf(self):
  583. g = _Lark("""start: A
  584. A: "a"* ("b"? "c".."e")+
  585. """)
  586. x = g.parse('abcde')
  587. x = g.parse('dd')
  588. def test_backslash(self):
  589. g = _Lark(r"""start: "\\" "a"
  590. """)
  591. x = g.parse(r'\a')
  592. g = _Lark(r"""start: /\\/ /a/
  593. """)
  594. x = g.parse(r'\a')
  595. def test_backslash2(self):
  596. g = _Lark(r"""start: "\"" "-"
  597. """)
  598. x = g.parse('"-')
  599. g = _Lark(r"""start: /\// /-/
  600. """)
  601. x = g.parse('/-')
  602. def test_special_chars(self):
  603. g = _Lark(r"""start: "\n"
  604. """)
  605. x = g.parse('\n')
  606. g = _Lark(r"""start: /\n/
  607. """)
  608. x = g.parse('\n')
  609. # def test_token_recurse(self):
  610. # g = _Lark("""start: A
  611. # A: B
  612. # B: A
  613. # """)
  614. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  615. def test_empty(self):
  616. # Fails an Earley implementation without special handling for empty rules,
  617. # or re-processing of already completed rules.
  618. g = _Lark(r"""start: _empty a "B"
  619. a: _empty "A"
  620. _empty:
  621. """)
  622. x = g.parse('AB')
  623. def test_regex_quote(self):
  624. g = r"""
  625. start: SINGLE_QUOTED_STRING | DOUBLE_QUOTED_STRING
  626. SINGLE_QUOTED_STRING : /'[^']*'/
  627. DOUBLE_QUOTED_STRING : /"[^"]*"/
  628. """
  629. g = _Lark(g)
  630. self.assertEqual( g.parse('"hello"').children, ['"hello"'])
  631. self.assertEqual( g.parse("'hello'").children, ["'hello'"])
  632. def test_lexer_token_limit(self):
  633. "Python has a stupid limit of 100 groups in a regular expression. Test that we handle this limitation"
  634. tokens = {'A%d'%i:'"%d"'%i for i in range(300)}
  635. g = _Lark("""start: %s
  636. %s""" % (' '.join(tokens), '\n'.join("%s: %s"%x for x in tokens.items())))
  637. def test_float_without_lexer(self):
  638. expected_error = UnexpectedInput if LEXER == 'dynamic' else UnexpectedToken
  639. if PARSER == 'cyk':
  640. expected_error = ParseError
  641. g = _Lark("""start: ["+"|"-"] float
  642. float: digit* "." digit+ exp?
  643. | digit+ exp
  644. exp: ("e"|"E") ["+"|"-"] digit+
  645. digit: "0"|"1"|"2"|"3"|"4"|"5"|"6"|"7"|"8"|"9"
  646. """)
  647. g.parse("1.2")
  648. g.parse("-.2e9")
  649. g.parse("+2e-9")
  650. self.assertRaises( expected_error, g.parse, "+2e-9e")
  651. def test_keep_all_tokens(self):
  652. l = _Lark("""start: "a"+ """, keep_all_tokens=True)
  653. tree = l.parse('aaa')
  654. self.assertEqual(tree.children, ['a', 'a', 'a'])
  655. def test_token_flags(self):
  656. l = _Lark("""!start: "a"i+
  657. """
  658. )
  659. tree = l.parse('aA')
  660. self.assertEqual(tree.children, ['a', 'A'])
  661. l = _Lark("""!start: /a/i+
  662. """
  663. )
  664. tree = l.parse('aA')
  665. self.assertEqual(tree.children, ['a', 'A'])
  666. # g = """!start: "a"i "a"
  667. # """
  668. # self.assertRaises(GrammarError, _Lark, g)
  669. # g = """!start: /a/i /a/
  670. # """
  671. # self.assertRaises(GrammarError, _Lark, g)
  672. g = """start: NAME "," "a"
  673. NAME: /[a-z_]/i /[a-z0-9_]/i*
  674. """
  675. l = _Lark(g)
  676. tree = l.parse('ab,a')
  677. self.assertEqual(tree.children, ['ab'])
  678. tree = l.parse('AB,a')
  679. self.assertEqual(tree.children, ['AB'])
  680. def test_token_flags3(self):
  681. l = _Lark("""!start: ABC+
  682. ABC: "abc"i
  683. """
  684. )
  685. tree = l.parse('aBcAbC')
  686. self.assertEqual(tree.children, ['aBc', 'AbC'])
  687. def test_token_flags2(self):
  688. g = """!start: ("a"i | /a/ /b/?)+
  689. """
  690. l = _Lark(g)
  691. tree = l.parse('aA')
  692. self.assertEqual(tree.children, ['a', 'A'])
  693. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  694. def test_twice_empty(self):
  695. g = """!start: [["A"]]
  696. """
  697. l = _Lark(g)
  698. tree = l.parse('A')
  699. self.assertEqual(tree.children, ['A'])
  700. tree = l.parse('')
  701. self.assertEqual(tree.children, [])
  702. def test_undefined_ignore(self):
  703. g = """!start: "A"
  704. %ignore B
  705. """
  706. self.assertRaises( GrammarError, _Lark, g)
  707. def test_alias_in_terminal(self):
  708. g = """start: TERM
  709. TERM: "a" -> alias
  710. """
  711. self.assertRaises( GrammarError, _Lark, g)
  712. def test_line_and_column(self):
  713. g = r"""!start: "A" bc "D"
  714. !bc: "B\nC"
  715. """
  716. l = _Lark(g)
  717. a, bc, d = l.parse("AB\nCD").children
  718. self.assertEqual(a.line, 1)
  719. self.assertEqual(a.column, 1)
  720. bc ,= bc.children
  721. self.assertEqual(bc.line, 1)
  722. self.assertEqual(bc.column, 2)
  723. self.assertEqual(d.line, 2)
  724. self.assertEqual(d.column, 2)
  725. if LEXER != 'dynamic':
  726. self.assertEqual(a.end_line, 1)
  727. self.assertEqual(a.end_column, 2)
  728. self.assertEqual(bc.end_line, 2)
  729. self.assertEqual(bc.end_column, 2)
  730. self.assertEqual(d.end_line, 2)
  731. self.assertEqual(d.end_column, 3)
  732. def test_reduce_cycle(self):
  733. """Tests an edge-condition in the LALR parser, in which a transition state looks exactly like the end state.
  734. It seems that the correct solution is to explicitely distinguish finalization in the reduce() function.
  735. """
  736. l = _Lark("""
  737. term: A
  738. | term term
  739. A: "a"
  740. """, start='term')
  741. tree = l.parse("aa")
  742. self.assertEqual(len(tree.children), 2)
  743. @unittest.skipIf(LEXER != 'standard', "Only standard lexers care about token priority")
  744. def test_lexer_prioritization(self):
  745. "Tests effect of priority on result"
  746. grammar = """
  747. start: A B | AB
  748. A.2: "a"
  749. B: "b"
  750. AB: "ab"
  751. """
  752. l = _Lark(grammar)
  753. res = l.parse("ab")
  754. self.assertEqual(res.children, ['a', 'b'])
  755. self.assertNotEqual(res.children, ['ab'])
  756. grammar = """
  757. start: A B | AB
  758. A: "a"
  759. B: "b"
  760. AB.3: "ab"
  761. """
  762. l = _Lark(grammar)
  763. res = l.parse("ab")
  764. self.assertNotEqual(res.children, ['a', 'b'])
  765. self.assertEqual(res.children, ['ab'])
  766. def test_import(self):
  767. grammar = """
  768. start: NUMBER WORD
  769. %import common.NUMBER
  770. %import common.WORD
  771. %import common.WS
  772. %ignore WS
  773. """
  774. l = _Lark(grammar)
  775. x = l.parse('12 elephants')
  776. self.assertEqual(x.children, ['12', 'elephants'])
  777. def test_relative_import(self):
  778. grammar = """
  779. start: NUMBER WORD
  780. %import .grammars.test.NUMBER
  781. %import common.WORD
  782. %import common.WS
  783. %ignore WS
  784. """
  785. l = _Lark(grammar)
  786. x = l.parse('12 lions')
  787. self.assertEqual(x.children, ['12', 'lions'])
  788. def test_multi_import(self):
  789. grammar = """
  790. start: NUMBER WORD
  791. %import common (NUMBER, WORD, WS)
  792. %ignore WS
  793. """
  794. l = _Lark(grammar)
  795. x = l.parse('12 toucans')
  796. self.assertEqual(x.children, ['12', 'toucans'])
  797. def test_relative_multi_import(self):
  798. grammar = """
  799. start: NUMBER WORD
  800. %import .grammars.test (NUMBER, WORD, WS)
  801. %ignore WS
  802. """
  803. l = _Lark(grammar)
  804. x = l.parse('12 capybaras')
  805. self.assertEqual(x.children, ['12', 'capybaras'])
  806. def test_import_errors(self):
  807. grammar = """
  808. start: NUMBER WORD
  809. %import .grammars.bad_test.NUMBER
  810. """
  811. self.assertRaises(IOError, _Lark, grammar)
  812. grammar = """
  813. start: NUMBER WORD
  814. %import bad_test.NUMBER
  815. """
  816. self.assertRaises(IOError, _Lark, grammar)
  817. @unittest.skipIf(PARSER != 'earley', "Currently only Earley supports priority in rules")
  818. def test_earley_prioritization(self):
  819. "Tests effect of priority on result"
  820. grammar = """
  821. start: a | b
  822. a.1: "a"
  823. b.2: "a"
  824. """
  825. # l = Lark(grammar, parser='earley', lexer='standard')
  826. l = _Lark(grammar)
  827. res = l.parse("a")
  828. self.assertEqual(res.children[0].data, 'b')
  829. grammar = """
  830. start: a | b
  831. a.2: "a"
  832. b.1: "a"
  833. """
  834. l = _Lark(grammar)
  835. # l = Lark(grammar, parser='earley', lexer='standard')
  836. res = l.parse("a")
  837. self.assertEqual(res.children[0].data, 'a')
  838. @unittest.skipIf(PARSER != 'earley', "Currently only Earley supports priority in rules")
  839. def test_earley_prioritization_sum(self):
  840. "Tests effect of priority on result"
  841. grammar = """
  842. start: ab_ b_ a_ | indirection
  843. indirection: a_ bb_ a_
  844. a_: "a"
  845. b_: "b"
  846. ab_: "ab"
  847. bb_.1: "bb"
  848. """
  849. l = Lark(grammar, priority="invert")
  850. res = l.parse('abba')
  851. self.assertEqual(''.join(child.data for child in res.children), 'ab_b_a_')
  852. grammar = """
  853. start: ab_ b_ a_ | indirection
  854. indirection: a_ bb_ a_
  855. a_: "a"
  856. b_: "b"
  857. ab_.1: "ab"
  858. bb_: "bb"
  859. """
  860. l = Lark(grammar, priority="invert")
  861. res = l.parse('abba')
  862. self.assertEqual(''.join(child.data for child in res.children), 'indirection')
  863. grammar = """
  864. start: ab_ b_ a_ | indirection
  865. indirection: a_ bb_ a_
  866. a_.2: "a"
  867. b_.1: "b"
  868. ab_.3: "ab"
  869. bb_.3: "bb"
  870. """
  871. l = Lark(grammar, priority="invert")
  872. res = l.parse('abba')
  873. self.assertEqual(''.join(child.data for child in res.children), 'ab_b_a_')
  874. grammar = """
  875. start: ab_ b_ a_ | indirection
  876. indirection: a_ bb_ a_
  877. a_.1: "a"
  878. b_.1: "b"
  879. ab_.4: "ab"
  880. bb_.3: "bb"
  881. """
  882. l = Lark(grammar, priority="invert")
  883. res = l.parse('abba')
  884. self.assertEqual(''.join(child.data for child in res.children), 'indirection')
  885. def test_utf8(self):
  886. g = u"""start: a
  887. a: "±a"
  888. """
  889. l = _Lark(g)
  890. self.assertEqual(l.parse(u'±a'), Tree('start', [Tree('a', [])]))
  891. g = u"""start: A
  892. A: "±a"
  893. """
  894. l = _Lark(g)
  895. self.assertEqual(l.parse(u'±a'), Tree('start', [u'\xb1a']))
  896. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  897. def test_ignore(self):
  898. grammar = r"""
  899. COMMENT: /(!|(\/\/))[^\n]*/
  900. %ignore COMMENT
  901. %import common.WS -> _WS
  902. %import common.INT
  903. start: "INT"i _WS+ INT _WS*
  904. """
  905. parser = _Lark(grammar)
  906. tree = parser.parse("int 1 ! This is a comment\n")
  907. self.assertEqual(tree.children, ['1'])
  908. tree = parser.parse("int 1 ! This is a comment") # A trailing ignore token can be tricky!
  909. self.assertEqual(tree.children, ['1'])
  910. parser = _Lark(r"""
  911. start : "a"*
  912. %ignore "b"
  913. """)
  914. tree = parser.parse("bb")
  915. self.assertEqual(tree.children, [])
  916. def test_regex_escaping(self):
  917. g = _Lark("start: /[ab]/")
  918. g.parse('a')
  919. g.parse('b')
  920. self.assertRaises( UnexpectedInput, g.parse, 'c')
  921. _Lark(r'start: /\w/').parse('a')
  922. g = _Lark(r'start: /\\w/')
  923. self.assertRaises( UnexpectedInput, g.parse, 'a')
  924. g.parse(r'\w')
  925. _Lark(r'start: /\[/').parse('[')
  926. _Lark(r'start: /\//').parse('/')
  927. _Lark(r'start: /\\/').parse('\\')
  928. _Lark(r'start: /\[ab]/').parse('[ab]')
  929. _Lark(r'start: /\\[ab]/').parse('\\a')
  930. _Lark(r'start: /\t/').parse('\t')
  931. _Lark(r'start: /\\t/').parse('\\t')
  932. _Lark(r'start: /\\\t/').parse('\\\t')
  933. _Lark(r'start: "\t"').parse('\t')
  934. _Lark(r'start: "\\t"').parse('\\t')
  935. _Lark(r'start: "\\\t"').parse('\\\t')
  936. def test_ranged_repeat_rules(self):
  937. g = u"""!start: "A"~3
  938. """
  939. l = _Lark(g)
  940. self.assertEqual(l.parse(u'AAA'), Tree('start', ["A", "A", "A"]))
  941. self.assertRaises(ParseError, l.parse, u'AA')
  942. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAA')
  943. g = u"""!start: "A"~0..2
  944. """
  945. if PARSER != 'cyk': # XXX CYK currently doesn't support empty grammars
  946. l = _Lark(g)
  947. self.assertEqual(l.parse(u''), Tree('start', []))
  948. self.assertEqual(l.parse(u'A'), Tree('start', ['A']))
  949. self.assertEqual(l.parse(u'AA'), Tree('start', ['A', 'A']))
  950. self.assertRaises((UnexpectedToken, UnexpectedInput), l.parse, u'AAA')
  951. g = u"""!start: "A"~3..2
  952. """
  953. self.assertRaises(GrammarError, _Lark, g)
  954. g = u"""!start: "A"~2..3 "B"~2
  955. """
  956. l = _Lark(g)
  957. self.assertEqual(l.parse(u'AABB'), Tree('start', ['A', 'A', 'B', 'B']))
  958. self.assertEqual(l.parse(u'AAABB'), Tree('start', ['A', 'A', 'A', 'B', 'B']))
  959. self.assertRaises(ParseError, l.parse, u'AAAB')
  960. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAABBB')
  961. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB')
  962. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAABB')
  963. def test_ranged_repeat_terms(self):
  964. g = u"""!start: AAA
  965. AAA: "A"~3
  966. """
  967. l = _Lark(g)
  968. self.assertEqual(l.parse(u'AAA'), Tree('start', ["AAA"]))
  969. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AA')
  970. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAA')
  971. g = u"""!start: AABB CC
  972. AABB: "A"~0..2 "B"~2
  973. CC: "C"~1..2
  974. """
  975. l = _Lark(g)
  976. self.assertEqual(l.parse(u'AABBCC'), Tree('start', ['AABB', 'CC']))
  977. self.assertEqual(l.parse(u'BBC'), Tree('start', ['BB', 'C']))
  978. self.assertEqual(l.parse(u'ABBCC'), Tree('start', ['ABB', 'CC']))
  979. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAB')
  980. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAABBB')
  981. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB')
  982. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAABB')
  983. @unittest.skipIf(PARSER=='earley', "Priority not handled correctly right now") # TODO XXX
  984. def test_priority_vs_embedded(self):
  985. g = """
  986. A.2: "a"
  987. WORD: ("a".."z")+
  988. start: (A | WORD)+
  989. """
  990. l = _Lark(g)
  991. t = l.parse('abc')
  992. self.assertEqual(t.children, ['a', 'bc'])
  993. self.assertEqual(t.children[0].type, 'A')
  994. def test_line_counting(self):
  995. p = _Lark("start: /[^x]+/")
  996. text = 'hello\nworld'
  997. t = p.parse(text)
  998. tok = t.children[0]
  999. self.assertEqual(tok, text)
  1000. self.assertEqual(tok.line, 1)
  1001. self.assertEqual(tok.column, 1)
  1002. if _LEXER != 'dynamic':
  1003. self.assertEqual(tok.end_line, 2)
  1004. self.assertEqual(tok.end_column, 6)
  1005. @unittest.skipIf(PARSER=='cyk', "Empty rules")
  1006. def test_empty_end(self):
  1007. p = _Lark("""
  1008. start: b c d
  1009. b: "B"
  1010. c: | "C"
  1011. d: | "D"
  1012. """)
  1013. res = p.parse('B')
  1014. self.assertEqual(len(res.children), 3)
  1015. @unittest.skipIf(PARSER=='cyk', "Empty rules")
  1016. def test_maybe_placeholders(self):
  1017. # Anonymous tokens shouldn't count
  1018. p = _Lark("""start: "a"? "b"? "c"? """, maybe_placeholders=True)
  1019. self.assertEqual(p.parse("").children, [])
  1020. # Anonymous tokens shouldn't count, other constructs should
  1021. p = _Lark("""start: A? "b"? _c?
  1022. A: "a"
  1023. _c: "c" """, maybe_placeholders=True)
  1024. self.assertEqual(p.parse("").children, [None])
  1025. p = _Lark("""!start: "a"? "b"? "c"? """, maybe_placeholders=True)
  1026. self.assertEqual(p.parse("").children, [None, None, None])
  1027. self.assertEqual(p.parse("a").children, ['a', None, None])
  1028. self.assertEqual(p.parse("b").children, [None, 'b', None])
  1029. self.assertEqual(p.parse("c").children, [None, None, 'c'])
  1030. self.assertEqual(p.parse("ab").children, ['a', 'b', None])
  1031. self.assertEqual(p.parse("ac").children, ['a', None, 'c'])
  1032. self.assertEqual(p.parse("bc").children, [None, 'b', 'c'])
  1033. self.assertEqual(p.parse("abc").children, ['a', 'b', 'c'])
  1034. p = _Lark("""!start: ("a"? "b" "c"?)+ """, maybe_placeholders=True)
  1035. self.assertEqual(p.parse("b").children, [None, 'b', None])
  1036. self.assertEqual(p.parse("bb").children, [None, 'b', None, None, 'b', None])
  1037. self.assertEqual(p.parse("abbc").children, ['a', 'b', None, None, 'b', 'c'])
  1038. self.assertEqual(p.parse("babbcabcb").children,
  1039. [None, 'b', None,
  1040. 'a', 'b', None,
  1041. None, 'b', 'c',
  1042. 'a', 'b', 'c',
  1043. None, 'b', None])
  1044. p = _Lark("""!start: "a"? "c"? "b"+ "a"? "d"? """, maybe_placeholders=True)
  1045. self.assertEqual(p.parse("bb").children, [None, None, 'b', 'b', None, None])
  1046. self.assertEqual(p.parse("bd").children, [None, None, 'b', None, 'd'])
  1047. self.assertEqual(p.parse("abba").children, ['a', None, 'b', 'b', 'a', None])
  1048. self.assertEqual(p.parse("cbbbb").children, [None, 'c', 'b', 'b', 'b', 'b', None, None])
  1049. def test_escaped_string(self):
  1050. "Tests common.ESCAPED_STRING"
  1051. grammar = r"""
  1052. start: ESCAPED_STRING+
  1053. %import common (WS_INLINE, ESCAPED_STRING)
  1054. %ignore WS_INLINE
  1055. """
  1056. parser = _Lark(grammar)
  1057. parser.parse(r'"\\" "b" "c"')
  1058. parser.parse(r'"That" "And a \"b"')
  1059. _NAME = "Test" + PARSER.capitalize() + LEXER.capitalize()
  1060. _TestParser.__name__ = _NAME
  1061. globals()[_NAME] = _TestParser
  1062. # Note: You still have to import them in __main__ for the tests to run
  1063. _TO_TEST = [
  1064. ('standard', 'earley'),
  1065. ('standard', 'cyk'),
  1066. ('dynamic', 'earley'),
  1067. ('dynamic_complete', 'earley'),
  1068. ('standard', 'lalr'),
  1069. ('contextual', 'lalr'),
  1070. # (None, 'earley'),
  1071. ]
  1072. for _LEXER, _PARSER in _TO_TEST:
  1073. _make_parser_test(_LEXER, _PARSER)
  1074. for _LEXER in ('dynamic', 'dynamic_complete'):
  1075. _make_full_earley_test(_LEXER)
  1076. if __name__ == '__main__':
  1077. unittest.main()