This repo contains code to mirror other repos. It also contains the code that is getting mirrored.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1158 lines
39 KiB

  1. # -*- coding: utf-8 -*-
  2. from __future__ import absolute_import
  3. import unittest
  4. import logging
  5. import os
  6. import sys
  7. try:
  8. from cStringIO import StringIO as cStringIO
  9. except ImportError:
  10. # Available only in Python 2.x, 3.x only has io.StringIO from below
  11. cStringIO = None
  12. from io import (
  13. StringIO as uStringIO,
  14. open,
  15. )
  16. logging.basicConfig(level=logging.INFO)
  17. from lark.lark import Lark
  18. from lark.common import GrammarError, ParseError, UnexpectedToken
  19. from lark.lexer import LexError, UnexpectedInput
  20. from lark.tree import Tree, Transformer
  21. __path__ = os.path.dirname(__file__)
  22. def _read(n, *args):
  23. with open(os.path.join(__path__, n), *args) as f:
  24. return f.read()
  25. class TestParsers(unittest.TestCase):
  26. def test_same_ast(self):
  27. "Tests that Earley and LALR parsers produce equal trees"
  28. g = Lark(r"""start: "(" name_list ("," "*" NAME)? ")"
  29. name_list: NAME | name_list "," NAME
  30. NAME: /\w+/ """, parser='lalr')
  31. l = g.parse('(a,b,c,*x)')
  32. g = Lark(r"""start: "(" name_list ("," "*" NAME)? ")"
  33. name_list: NAME | name_list "," NAME
  34. NAME: /\w/+ """)
  35. l2 = g.parse('(a,b,c,*x)')
  36. assert l == l2, '%s != %s' % (l.pretty(), l2.pretty())
  37. def test_infinite_recurse(self):
  38. g = """start: a
  39. a: a | "a"
  40. """
  41. self.assertRaises(GrammarError, Lark, g, parser='lalr')
  42. l = Lark(g, parser='earley', lexer=None)
  43. self.assertRaises(ParseError, l.parse, 'a')
  44. l = Lark(g, parser='earley', lexer='dynamic')
  45. self.assertRaises(ParseError, l.parse, 'a')
  46. def test_propagate_positions(self):
  47. g = Lark("""start: a
  48. a: "a"
  49. """, propagate_positions=True)
  50. r = g.parse('a')
  51. self.assertEqual( r.children[0].line, 1 )
  52. def test_expand1(self):
  53. g = Lark("""start: a
  54. ?a: b
  55. b: "x"
  56. """)
  57. r = g.parse('x')
  58. self.assertEqual( r.children[0].data, "b" )
  59. g = Lark("""start: a
  60. ?a: b -> c
  61. b: "x"
  62. """)
  63. r = g.parse('x')
  64. self.assertEqual( r.children[0].data, "c" )
  65. g = Lark("""start: a
  66. ?a: B -> c
  67. B: "x"
  68. """)
  69. self.assertEqual( r.children[0].data, "c" )
  70. g = Lark("""start: a
  71. ?a: b b -> c
  72. b: "x"
  73. """)
  74. r = g.parse('xx')
  75. self.assertEqual( r.children[0].data, "c" )
  76. def test_embedded_transformer(self):
  77. class T(Transformer):
  78. def a(self, children):
  79. return "<a>"
  80. def b(self, children):
  81. return "<b>"
  82. def c(self, children):
  83. return "<c>"
  84. # Test regular
  85. g = Lark("""start: a
  86. a : "x"
  87. """, parser='lalr')
  88. r = T().transform(g.parse("x"))
  89. self.assertEqual( r.children, ["<a>"] )
  90. g = Lark("""start: a
  91. a : "x"
  92. """, parser='lalr', transformer=T())
  93. r = g.parse("x")
  94. self.assertEqual( r.children, ["<a>"] )
  95. # Test Expand1
  96. g = Lark("""start: a
  97. ?a : b
  98. b : "x"
  99. """, parser='lalr')
  100. r = T().transform(g.parse("x"))
  101. self.assertEqual( r.children, ["<b>"] )
  102. g = Lark("""start: a
  103. ?a : b
  104. b : "x"
  105. """, parser='lalr', transformer=T())
  106. r = g.parse("x")
  107. self.assertEqual( r.children, ["<b>"] )
  108. # Test Expand1 -> Alias
  109. g = Lark("""start: a
  110. ?a : b b -> c
  111. b : "x"
  112. """, parser='lalr')
  113. r = T().transform(g.parse("xx"))
  114. self.assertEqual( r.children, ["<c>"] )
  115. g = Lark("""start: a
  116. ?a : b b -> c
  117. b : "x"
  118. """, parser='lalr', transformer=T())
  119. r = g.parse("xx")
  120. self.assertEqual( r.children, ["<c>"] )
  121. def _make_full_earley_test(LEXER):
  122. class _TestFullEarley(unittest.TestCase):
  123. def test_anon_in_scanless(self):
  124. # Fails an Earley implementation without special handling for empty rules,
  125. # or re-processing of already completed rules.
  126. g = Lark(r"""start: B
  127. B: ("ab"|/[^b]/)+
  128. """, lexer=LEXER)
  129. self.assertEqual( g.parse('abc').children[0], 'abc')
  130. def test_earley_scanless(self):
  131. g = Lark("""start: A "b" c
  132. A: "a"+
  133. c: "abc"
  134. """, parser="earley", lexer=LEXER)
  135. x = g.parse('aaaababc')
  136. def test_earley_scanless2(self):
  137. grammar = """
  138. start: statement+
  139. statement: "r"
  140. | "c" /[a-z]/+
  141. %ignore " "
  142. """
  143. program = """c b r"""
  144. l = Lark(grammar, parser='earley', lexer=LEXER)
  145. l.parse(program)
  146. def test_earley_scanless3(self):
  147. "Tests prioritization and disambiguation for pseudo-terminals (there should be only one result)"
  148. grammar = """
  149. start: A A
  150. A: "a"+
  151. """
  152. l = Lark(grammar, parser='earley', lexer=LEXER)
  153. res = l.parse("aaa")
  154. self.assertEqual(res.children, ['aa', 'a'])
  155. def test_earley_scanless4(self):
  156. grammar = """
  157. start: A A?
  158. A: "a"+
  159. """
  160. l = Lark(grammar, parser='earley', lexer=LEXER)
  161. res = l.parse("aaa")
  162. self.assertEqual(res.children, ['aaa'])
  163. def test_earley_repeating_empty(self):
  164. # This was a sneaky bug!
  165. grammar = """
  166. !start: "a" empty empty "b"
  167. empty: empty2
  168. empty2:
  169. """
  170. parser = Lark(grammar, parser='earley', lexer=LEXER)
  171. res = parser.parse('ab')
  172. empty_tree = Tree('empty', [Tree('empty2', [])])
  173. self.assertSequenceEqual(res.children, ['a', empty_tree, empty_tree, 'b'])
  174. def test_earley_explicit_ambiguity(self):
  175. # This was a sneaky bug!
  176. grammar = """
  177. start: a b | ab
  178. a: "a"
  179. b: "b"
  180. ab: "ab"
  181. """
  182. parser = Lark(grammar, parser='earley', lexer=LEXER, ambiguity='explicit')
  183. res = parser.parse('ab')
  184. self.assertEqual( res.data, '_ambig')
  185. self.assertEqual( len(res.children), 2)
  186. def test_ambiguity1(self):
  187. grammar = """
  188. start: cd+ "e"
  189. !cd: "c"
  190. | "d"
  191. | "cd"
  192. """
  193. l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER)
  194. x = l.parse('cde')
  195. assert x.data == '_ambig', x
  196. assert len(x.children) == 2
  197. @unittest.skipIf(LEXER==None, "BUG in scanless parsing!") # TODO fix bug!
  198. def test_fruitflies_ambig(self):
  199. grammar = """
  200. start: noun verb noun -> simple
  201. | noun verb "like" noun -> comparative
  202. noun: adj? NOUN
  203. verb: VERB
  204. adj: ADJ
  205. NOUN: "flies" | "bananas" | "fruit"
  206. VERB: "like" | "flies"
  207. ADJ: "fruit"
  208. %import common.WS
  209. %ignore WS
  210. """
  211. parser = Lark(grammar, ambiguity='explicit', lexer=LEXER)
  212. res = parser.parse('fruit flies like bananas')
  213. expected = Tree('_ambig', [
  214. Tree('comparative', [
  215. Tree('noun', ['fruit']),
  216. Tree('verb', ['flies']),
  217. Tree('noun', ['bananas'])
  218. ]),
  219. Tree('simple', [
  220. Tree('noun', [Tree('adj', ['fruit']), 'flies']),
  221. Tree('verb', ['like']),
  222. Tree('noun', ['bananas'])
  223. ])
  224. ])
  225. # print res.pretty()
  226. # print expected.pretty()
  227. self.assertEqual(res, expected)
  228. # @unittest.skipIf(LEXER=='dynamic', "Not implemented in Dynamic Earley yet") # TODO
  229. # def test_not_all_derivations(self):
  230. # grammar = """
  231. # start: cd+ "e"
  232. # !cd: "c"
  233. # | "d"
  234. # | "cd"
  235. # """
  236. # l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER, earley__all_derivations=False)
  237. # x = l.parse('cde')
  238. # assert x.data != '_ambig', x
  239. # assert len(x.children) == 1
  240. _NAME = "TestFullEarley" + (LEXER or 'Scanless').capitalize()
  241. _TestFullEarley.__name__ = _NAME
  242. globals()[_NAME] = _TestFullEarley
  243. def _make_parser_test(LEXER, PARSER):
  244. def _Lark(grammar, **kwargs):
  245. return Lark(grammar, lexer=LEXER, parser=PARSER, **kwargs)
  246. class _TestParser(unittest.TestCase):
  247. def test_basic1(self):
  248. g = _Lark("""start: a+ b a* "b" a*
  249. b: "b"
  250. a: "a"
  251. """)
  252. r = g.parse('aaabaab')
  253. self.assertEqual( ''.join(x.data for x in r.children), 'aaabaa' )
  254. r = g.parse('aaabaaba')
  255. self.assertEqual( ''.join(x.data for x in r.children), 'aaabaaa' )
  256. self.assertRaises(ParseError, g.parse, 'aaabaa')
  257. def test_basic2(self):
  258. # Multiple parsers and colliding tokens
  259. g = _Lark("""start: B A
  260. B: "12"
  261. A: "1" """)
  262. g2 = _Lark("""start: B A
  263. B: "12"
  264. A: "2" """)
  265. x = g.parse('121')
  266. assert x.data == 'start' and x.children == ['12', '1'], x
  267. x = g2.parse('122')
  268. assert x.data == 'start' and x.children == ['12', '2'], x
  269. @unittest.skipIf(cStringIO is None, "cStringIO not available")
  270. def test_stringio_bytes(self):
  271. """Verify that a Lark can be created from file-like objects other than Python's standard 'file' object"""
  272. _Lark(cStringIO(b'start: a+ b a* "b" a*\n b: "b"\n a: "a" '))
  273. def test_stringio_unicode(self):
  274. """Verify that a Lark can be created from file-like objects other than Python's standard 'file' object"""
  275. _Lark(uStringIO(u'start: a+ b a* "b" a*\n b: "b"\n a: "a" '))
  276. def test_unicode(self):
  277. g = _Lark(u"""start: UNIA UNIB UNIA
  278. UNIA: /\xa3/
  279. UNIB: /\u0101/
  280. """)
  281. g.parse(u'\xa3\u0101\u00a3')
  282. @unittest.skipIf(LEXER is None, "Regexps >1 not supported with scanless parsing")
  283. def test_unicode2(self):
  284. g = _Lark(r"""start: UNIA UNIB UNIA UNIC
  285. UNIA: /\xa3/
  286. UNIB: "a\u0101b\ "
  287. UNIC: /a?\u0101c\n/
  288. """)
  289. g.parse(u'\xa3a\u0101b\\ \u00a3\u0101c\n')
  290. def test_unicode3(self):
  291. g = _Lark(r"""start: UNIA UNIB UNIA UNIC
  292. UNIA: /\xa3/
  293. UNIB: "\u0101"
  294. UNIC: /\u0203/ /\n/
  295. """)
  296. g.parse(u'\xa3\u0101\u00a3\u0203\n')
  297. @unittest.skipIf(PARSER == 'cyk', "Takes forever")
  298. def test_stack_for_ebnf(self):
  299. """Verify that stack depth isn't an issue for EBNF grammars"""
  300. g = _Lark(r"""start: a+
  301. a : "a" """)
  302. g.parse("a" * (sys.getrecursionlimit()*2 ))
  303. def test_expand1_lists_with_one_item(self):
  304. g = _Lark(r"""start: list
  305. ?list: item+
  306. item : A
  307. A: "a"
  308. """)
  309. r = g.parse("a")
  310. # because 'list' is an expand-if-contains-one rule and we only provided one element it should have expanded to 'item'
  311. self.assertSequenceEqual([subtree.data for subtree in r.children], ('item',))
  312. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  313. self.assertEqual(len(r.children), 1)
  314. def test_expand1_lists_with_one_item_2(self):
  315. g = _Lark(r"""start: list
  316. ?list: item+ "!"
  317. item : A
  318. A: "a"
  319. """)
  320. r = g.parse("a!")
  321. # because 'list' is an expand-if-contains-one rule and we only provided one element it should have expanded to 'item'
  322. self.assertSequenceEqual([subtree.data for subtree in r.children], ('item',))
  323. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  324. self.assertEqual(len(r.children), 1)
  325. def test_dont_expand1_lists_with_multiple_items(self):
  326. g = _Lark(r"""start: list
  327. ?list: item+
  328. item : A
  329. A: "a"
  330. """)
  331. r = g.parse("aa")
  332. # because 'list' is an expand-if-contains-one rule and we've provided more than one element it should *not* have expanded
  333. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  334. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  335. self.assertEqual(len(r.children), 1)
  336. # Sanity check: verify that 'list' contains the two 'item's we've given it
  337. [list] = r.children
  338. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  339. def test_dont_expand1_lists_with_multiple_items_2(self):
  340. g = _Lark(r"""start: list
  341. ?list: item+ "!"
  342. item : A
  343. A: "a"
  344. """)
  345. r = g.parse("aa!")
  346. # because 'list' is an expand-if-contains-one rule and we've provided more than one element it should *not* have expanded
  347. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  348. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  349. self.assertEqual(len(r.children), 1)
  350. # Sanity check: verify that 'list' contains the two 'item's we've given it
  351. [list] = r.children
  352. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  353. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  354. def test_empty_expand1_list(self):
  355. g = _Lark(r"""start: list
  356. ?list: item*
  357. item : A
  358. A: "a"
  359. """)
  360. r = g.parse("")
  361. # because 'list' is an expand-if-contains-one rule and we've provided less than one element (i.e. none) it should *not* have expanded
  362. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  363. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  364. self.assertEqual(len(r.children), 1)
  365. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  366. [list] = r.children
  367. self.assertSequenceEqual([item.data for item in list.children], ())
  368. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  369. def test_empty_expand1_list_2(self):
  370. g = _Lark(r"""start: list
  371. ?list: item* "!"?
  372. item : A
  373. A: "a"
  374. """)
  375. r = g.parse("")
  376. # because 'list' is an expand-if-contains-one rule and we've provided less than one element (i.e. none) it should *not* have expanded
  377. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  378. # regardless of the amount of items: there should be only *one* child in 'start' because 'list' isn't an expand-all rule
  379. self.assertEqual(len(r.children), 1)
  380. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  381. [list] = r.children
  382. self.assertSequenceEqual([item.data for item in list.children], ())
  383. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  384. def test_empty_flatten_list(self):
  385. g = _Lark(r"""start: list
  386. list: | item "," list
  387. item : A
  388. A: "a"
  389. """)
  390. r = g.parse("")
  391. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  392. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  393. # Sanity check: verify that 'list' contains no 'item's as we've given it none
  394. [list] = r.children
  395. self.assertSequenceEqual([item.data for item in list.children], ())
  396. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  397. def test_single_item_flatten_list(self):
  398. g = _Lark(r"""start: list
  399. list: | item "," list
  400. item : A
  401. A: "a"
  402. """)
  403. r = g.parse("a,")
  404. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  405. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  406. # Sanity check: verify that 'list' contains exactly the one 'item' we've given it
  407. [list] = r.children
  408. self.assertSequenceEqual([item.data for item in list.children], ('item',))
  409. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  410. def test_multiple_item_flatten_list(self):
  411. g = _Lark(r"""start: list
  412. #list: | item "," list
  413. item : A
  414. A: "a"
  415. """)
  416. r = g.parse("a,a,")
  417. # Because 'list' is a flatten rule it's top-level element should *never* be expanded
  418. self.assertSequenceEqual([subtree.data for subtree in r.children], ('list',))
  419. # Sanity check: verify that 'list' contains exactly the two 'item's we've given it
  420. [list] = r.children
  421. self.assertSequenceEqual([item.data for item in list.children], ('item', 'item'))
  422. @unittest.skipIf(True, "Flattening list isn't implemented (and may never be)")
  423. def test_recurse_flatten(self):
  424. """Verify that stack depth doesn't get exceeded on recursive rules marked for flattening."""
  425. g = _Lark(r"""start: a | start a
  426. a : A
  427. A : "a" """)
  428. # Force PLY to write to the debug log, but prevent writing it to the terminal (uses repr() on the half-built
  429. # STree data structures, which uses recursion).
  430. g.parse("a" * (sys.getrecursionlimit() // 4))
  431. def test_token_collision(self):
  432. g = _Lark(r"""start: "Hello" NAME
  433. NAME: /\w/+
  434. %ignore " "
  435. """)
  436. x = g.parse('Hello World')
  437. self.assertSequenceEqual(x.children, ['World'])
  438. x = g.parse('Hello HelloWorld')
  439. self.assertSequenceEqual(x.children, ['HelloWorld'])
  440. def test_token_collision_WS(self):
  441. g = _Lark(r"""start: "Hello" NAME
  442. NAME: /\w/+
  443. %import common.WS
  444. %ignore WS
  445. """)
  446. x = g.parse('Hello World')
  447. self.assertSequenceEqual(x.children, ['World'])
  448. x = g.parse('Hello HelloWorld')
  449. self.assertSequenceEqual(x.children, ['HelloWorld'])
  450. @unittest.skipIf(LEXER is None, "Known bug with scanless parsing") # TODO
  451. def test_token_collision2(self):
  452. # NOTE: This test reveals a bug in token reconstruction in Scanless Earley
  453. # I probably need to re-write grammar transformation
  454. g = _Lark("""
  455. !start: "starts"
  456. %import common.LCASE_LETTER
  457. """)
  458. x = g.parse("starts")
  459. self.assertSequenceEqual(x.children, ['starts'])
  460. # def test_string_priority(self):
  461. # g = _Lark("""start: (A | /a?bb/)+
  462. # A: "a" """)
  463. # x = g.parse('abb')
  464. # self.assertEqual(len(x.children), 2)
  465. # # This parse raises an exception because the lexer will always try to consume
  466. # # "a" first and will never match the regular expression
  467. # # This behavior is subject to change!!
  468. # # Thie won't happen with ambiguity handling.
  469. # g = _Lark("""start: (A | /a?ab/)+
  470. # A: "a" """)
  471. # self.assertRaises(LexError, g.parse, 'aab')
  472. def test_undefined_rule(self):
  473. self.assertRaises(GrammarError, _Lark, """start: a""")
  474. def test_undefined_token(self):
  475. self.assertRaises(GrammarError, _Lark, """start: A""")
  476. def test_rule_collision(self):
  477. g = _Lark("""start: "a"+ "b"
  478. | "a"+ """)
  479. x = g.parse('aaaa')
  480. x = g.parse('aaaab')
  481. def test_rule_collision2(self):
  482. g = _Lark("""start: "a"* "b"
  483. | "a"+ """)
  484. x = g.parse('aaaa')
  485. x = g.parse('aaaab')
  486. x = g.parse('b')
  487. @unittest.skipIf(LEXER in (None, 'dynamic'), "Known bug with scanless parsing") # TODO
  488. def test_token_not_anon(self):
  489. """Tests that "a" is matched as A, rather than an anonymous token.
  490. That means that "a" is not filtered out, despite being an 'immediate string'.
  491. Whether or not this is the intuitive behavior, I'm not sure yet.
  492. Perhaps the right thing to do is report a collision (if such is relevant)
  493. -Erez
  494. """
  495. g = _Lark("""start: "a"
  496. A: "a" """)
  497. x = g.parse('a')
  498. self.assertEqual(len(x.children), 1, '"a" should not be considered anonymous')
  499. self.assertEqual(x.children[0].type, "A")
  500. g = _Lark("""start: /a/
  501. A: /a/ """)
  502. x = g.parse('a')
  503. self.assertEqual(len(x.children), 1, '/a/ should not be considered anonymous')
  504. self.assertEqual(x.children[0].type, "A")
  505. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  506. def test_maybe(self):
  507. g = _Lark("""start: ["a"] """)
  508. x = g.parse('a')
  509. x = g.parse('')
  510. def test_start(self):
  511. g = _Lark("""a: "a" a? """, start='a')
  512. x = g.parse('a')
  513. x = g.parse('aa')
  514. x = g.parse('aaa')
  515. def test_alias(self):
  516. g = _Lark("""start: "a" -> b """)
  517. x = g.parse('a')
  518. self.assertEqual(x.data, "b")
  519. def test_token_ebnf(self):
  520. g = _Lark("""start: A
  521. A: "a"* ("b"? "c".."e")+
  522. """)
  523. x = g.parse('abcde')
  524. x = g.parse('dd')
  525. def test_backslash(self):
  526. g = _Lark(r"""start: "\\" "a"
  527. """)
  528. x = g.parse(r'\a')
  529. g = _Lark(r"""start: /\\/ /a/
  530. """)
  531. x = g.parse(r'\a')
  532. def test_special_chars(self):
  533. g = _Lark(r"""start: "\n"
  534. """)
  535. x = g.parse('\n')
  536. g = _Lark(r"""start: /\n/
  537. """)
  538. x = g.parse('\n')
  539. def test_backslash2(self):
  540. g = _Lark(r"""start: "\"" "-"
  541. """)
  542. x = g.parse('"-')
  543. g = _Lark(r"""start: /\// /-/
  544. """)
  545. x = g.parse('/-')
  546. # def test_token_recurse(self):
  547. # g = _Lark("""start: A
  548. # A: B
  549. # B: A
  550. # """)
  551. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  552. def test_empty(self):
  553. # Fails an Earley implementation without special handling for empty rules,
  554. # or re-processing of already completed rules.
  555. g = _Lark(r"""start: _empty a "B"
  556. a: _empty "A"
  557. _empty:
  558. """)
  559. x = g.parse('AB')
  560. @unittest.skipIf(LEXER == None, "Scanless can't handle regexps")
  561. def test_regex_quote(self):
  562. g = r"""
  563. start: SINGLE_QUOTED_STRING | DOUBLE_QUOTED_STRING
  564. SINGLE_QUOTED_STRING : /'[^']*'/
  565. DOUBLE_QUOTED_STRING : /"[^"]*"/
  566. """
  567. g = _Lark(g)
  568. self.assertEqual( g.parse('"hello"').children, ['"hello"'])
  569. self.assertEqual( g.parse("'hello'").children, ["'hello'"])
  570. def test_lexer_token_limit(self):
  571. "Python has a stupid limit of 100 groups in a regular expression. Test that we handle this limitation"
  572. tokens = {'A%d'%i:'"%d"'%i for i in range(300)}
  573. g = _Lark("""start: %s
  574. %s""" % (' '.join(tokens), '\n'.join("%s: %s"%x for x in tokens.items())))
  575. def test_float_without_lexer(self):
  576. expected_error = UnexpectedInput if LEXER == 'dynamic' else UnexpectedToken
  577. if PARSER == 'cyk':
  578. expected_error = ParseError
  579. g = _Lark("""start: ["+"|"-"] float
  580. float: digit* "." digit+ exp?
  581. | digit+ exp
  582. exp: ("e"|"E") ["+"|"-"] digit+
  583. digit: "0"|"1"|"2"|"3"|"4"|"5"|"6"|"7"|"8"|"9"
  584. """)
  585. g.parse("1.2")
  586. g.parse("-.2e9")
  587. g.parse("+2e-9")
  588. self.assertRaises( expected_error, g.parse, "+2e-9e")
  589. def test_keep_all_tokens(self):
  590. l = _Lark("""start: "a"+ """, keep_all_tokens=True)
  591. tree = l.parse('aaa')
  592. self.assertEqual(tree.children, ['a', 'a', 'a'])
  593. def test_token_flags(self):
  594. l = _Lark("""!start: "a"i+
  595. """
  596. )
  597. tree = l.parse('aA')
  598. self.assertEqual(tree.children, ['a', 'A'])
  599. l = _Lark("""!start: /a/i+
  600. """
  601. )
  602. tree = l.parse('aA')
  603. self.assertEqual(tree.children, ['a', 'A'])
  604. # g = """!start: "a"i "a"
  605. # """
  606. # self.assertRaises(GrammarError, _Lark, g)
  607. # g = """!start: /a/i /a/
  608. # """
  609. # self.assertRaises(GrammarError, _Lark, g)
  610. g = """start: NAME "," "a"
  611. NAME: /[a-z_]/i /[a-z0-9_]/i*
  612. """
  613. l = _Lark(g)
  614. tree = l.parse('ab,a')
  615. self.assertEqual(tree.children, ['ab'])
  616. tree = l.parse('AB,a')
  617. self.assertEqual(tree.children, ['AB'])
  618. def test_token_flags3(self):
  619. l = _Lark("""!start: ABC+
  620. ABC: "abc"i
  621. """
  622. )
  623. tree = l.parse('aBcAbC')
  624. self.assertEqual(tree.children, ['aBc', 'AbC'])
  625. def test_token_flags2(self):
  626. g = """!start: ("a"i | /a/ /b/?)+
  627. """
  628. l = _Lark(g)
  629. tree = l.parse('aA')
  630. self.assertEqual(tree.children, ['a', 'A'])
  631. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  632. def test_twice_empty(self):
  633. g = """!start: [["A"]]
  634. """
  635. l = _Lark(g)
  636. tree = l.parse('A')
  637. self.assertEqual(tree.children, ['A'])
  638. tree = l.parse('')
  639. self.assertEqual(tree.children, [])
  640. def test_undefined_ignore(self):
  641. g = """!start: "A"
  642. %ignore B
  643. """
  644. self.assertRaises( GrammarError, _Lark, g)
  645. @unittest.skipIf(LEXER==None, "TODO: Fix scanless parsing or get rid of it") # TODO
  646. def test_line_and_column(self):
  647. g = r"""!start: "A" bc "D"
  648. !bc: "B\nC"
  649. """
  650. l = _Lark(g)
  651. a, bc, d = l.parse("AB\nCD").children
  652. self.assertEqual(a.line, 1)
  653. self.assertEqual(a.column, 0)
  654. bc ,= bc.children
  655. self.assertEqual(bc.line, 1)
  656. self.assertEqual(bc.column, 1)
  657. self.assertEqual(d.line, 2)
  658. self.assertEqual(d.column, 1)
  659. if LEXER != 'dynamic':
  660. self.assertEqual(a.end_line, 1)
  661. self.assertEqual(a.end_column, 1)
  662. self.assertEqual(bc.end_line, 2)
  663. self.assertEqual(bc.end_column, 1)
  664. self.assertEqual(d.end_line, 2)
  665. self.assertEqual(d.end_column, 2)
  666. def test_reduce_cycle(self):
  667. """Tests an edge-condition in the LALR parser, in which a transition state looks exactly like the end state.
  668. It seems that the correct solution is to explicitely distinguish finalization in the reduce() function.
  669. """
  670. l = _Lark("""
  671. term: A
  672. | term term
  673. A: "a"
  674. """, start='term')
  675. tree = l.parse("aa")
  676. self.assertEqual(len(tree.children), 2)
  677. @unittest.skipIf(LEXER != 'standard', "Only standard lexers care about token priority")
  678. def test_lexer_prioritization(self):
  679. "Tests effect of priority on result"
  680. grammar = """
  681. start: A B | AB
  682. A.2: "a"
  683. B: "b"
  684. AB: "ab"
  685. """
  686. l = _Lark(grammar)
  687. res = l.parse("ab")
  688. self.assertEqual(res.children, ['a', 'b'])
  689. self.assertNotEqual(res.children, ['ab'])
  690. grammar = """
  691. start: A B | AB
  692. A: "a"
  693. B: "b"
  694. AB.3: "ab"
  695. """
  696. l = _Lark(grammar)
  697. res = l.parse("ab")
  698. self.assertNotEqual(res.children, ['a', 'b'])
  699. self.assertEqual(res.children, ['ab'])
  700. def test_import(self):
  701. grammar = """
  702. start: NUMBER WORD
  703. %import common.NUMBER
  704. %import common.WORD
  705. %import common.WS
  706. %ignore WS
  707. """
  708. l = _Lark(grammar)
  709. x = l.parse('12 elephants')
  710. self.assertEqual(x.children, ['12', 'elephants'])
  711. @unittest.skipIf(PARSER != 'earley', "Currently only Earley supports priority in rules")
  712. def test_earley_prioritization(self):
  713. "Tests effect of priority on result"
  714. grammar = """
  715. start: a | b
  716. a.1: "a"
  717. b.2: "a"
  718. """
  719. # l = Lark(grammar, parser='earley', lexer='standard')
  720. l = _Lark(grammar)
  721. res = l.parse("a")
  722. self.assertEqual(res.children[0].data, 'b')
  723. grammar = """
  724. start: a | b
  725. a.2: "a"
  726. b.1: "a"
  727. """
  728. l = _Lark(grammar)
  729. # l = Lark(grammar, parser='earley', lexer='standard')
  730. res = l.parse("a")
  731. self.assertEqual(res.children[0].data, 'a')
  732. @unittest.skipIf(PARSER != 'earley', "Currently only Earley supports priority in rules")
  733. def test_earley_prioritization_sum(self):
  734. "Tests effect of priority on result"
  735. grammar = """
  736. start: ab_ b_ a_ | indirection
  737. indirection: a_ bb_ a_
  738. a_: "a"
  739. b_: "b"
  740. ab_: "ab"
  741. bb_.1: "bb"
  742. """
  743. l = _Lark(grammar, ambiguity='resolve__antiscore_sum')
  744. res = l.parse('abba')
  745. self.assertEqual(''.join(child.data for child in res.children), 'ab_b_a_')
  746. grammar = """
  747. start: ab_ b_ a_ | indirection
  748. indirection: a_ bb_ a_
  749. a_: "a"
  750. b_: "b"
  751. ab_.1: "ab"
  752. bb_: "bb"
  753. """
  754. l = Lark(grammar, parser='earley', ambiguity='resolve__antiscore_sum')
  755. res = l.parse('abba')
  756. self.assertEqual(''.join(child.data for child in res.children), 'indirection')
  757. grammar = """
  758. start: ab_ b_ a_ | indirection
  759. indirection: a_ bb_ a_
  760. a_.2: "a"
  761. b_.1: "b"
  762. ab_.3: "ab"
  763. bb_.3: "bb"
  764. """
  765. l = Lark(grammar, parser='earley', ambiguity='resolve__antiscore_sum')
  766. res = l.parse('abba')
  767. self.assertEqual(''.join(child.data for child in res.children), 'ab_b_a_')
  768. grammar = """
  769. start: ab_ b_ a_ | indirection
  770. indirection: a_ bb_ a_
  771. a_.1: "a"
  772. b_.1: "b"
  773. ab_.4: "ab"
  774. bb_.3: "bb"
  775. """
  776. l = Lark(grammar, parser='earley', ambiguity='resolve__antiscore_sum')
  777. res = l.parse('abba')
  778. self.assertEqual(''.join(child.data for child in res.children), 'indirection')
  779. def test_utf8(self):
  780. g = u"""start: a
  781. a: "±a"
  782. """
  783. l = _Lark(g)
  784. self.assertEqual(l.parse(u'±a'), Tree('start', [Tree('a', [])]))
  785. g = u"""start: A
  786. A: "±a"
  787. """
  788. l = _Lark(g)
  789. self.assertEqual(l.parse(u'±a'), Tree('start', [u'\xb1a']))
  790. @unittest.skipIf(LEXER==None, "Scanless doesn't support regular expressions")
  791. @unittest.skipIf(PARSER == 'cyk', "No empty rules")
  792. def test_ignore(self):
  793. grammar = r"""
  794. COMMENT: /(!|(\/\/))[^\n]*/
  795. %ignore COMMENT
  796. %import common.WS -> _WS
  797. %import common.INT
  798. start: "INT"i _WS+ INT _WS*
  799. """
  800. parser = _Lark(grammar)
  801. tree = parser.parse("int 1 ! This is a comment\n")
  802. self.assertEqual(tree.children, ['1'])
  803. tree = parser.parse("int 1 ! This is a comment") # A trailing ignore token can be tricky!
  804. self.assertEqual(tree.children, ['1'])
  805. parser = _Lark(r"""
  806. start : "a"*
  807. %ignore "b"
  808. """)
  809. tree = parser.parse("bb")
  810. self.assertEqual(tree.children, [])
  811. @unittest.skipIf(LEXER==None, "Scanless doesn't support regular expressions")
  812. def test_regex_escaping(self):
  813. g = _Lark("start: /[ab]/")
  814. g.parse('a')
  815. g.parse('b')
  816. self.assertRaises( UnexpectedInput, g.parse, 'c')
  817. _Lark(r'start: /\w/').parse('a')
  818. g = _Lark(r'start: /\\w/')
  819. self.assertRaises( UnexpectedInput, g.parse, 'a')
  820. g.parse(r'\w')
  821. _Lark(r'start: /\[/').parse('[')
  822. _Lark(r'start: /\//').parse('/')
  823. _Lark(r'start: /\\/').parse('\\')
  824. _Lark(r'start: /\[ab]/').parse('[ab]')
  825. _Lark(r'start: /\\[ab]/').parse('\\a')
  826. _Lark(r'start: /\t/').parse('\t')
  827. _Lark(r'start: /\\t/').parse('\\t')
  828. _Lark(r'start: /\\\t/').parse('\\\t')
  829. _Lark(r'start: "\t"').parse('\t')
  830. _Lark(r'start: "\\t"').parse('\\t')
  831. _Lark(r'start: "\\\t"').parse('\\\t')
  832. def test_ranged_repeat_rules(self):
  833. g = u"""!start: "A"~3
  834. """
  835. l = _Lark(g)
  836. self.assertEqual(l.parse(u'AAA'), Tree('start', ["A", "A", "A"]))
  837. self.assertRaises(ParseError, l.parse, u'AA')
  838. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAA')
  839. g = u"""!start: "A"~0..2
  840. """
  841. if PARSER != 'cyk': # XXX CYK currently doesn't support empty grammars
  842. l = _Lark(g)
  843. self.assertEqual(l.parse(u''), Tree('start', []))
  844. self.assertEqual(l.parse(u'A'), Tree('start', ['A']))
  845. self.assertEqual(l.parse(u'AA'), Tree('start', ['A', 'A']))
  846. self.assertRaises((UnexpectedToken, UnexpectedInput), l.parse, u'AAA')
  847. g = u"""!start: "A"~3..2
  848. """
  849. self.assertRaises(GrammarError, _Lark, g)
  850. g = u"""!start: "A"~2..3 "B"~2
  851. """
  852. l = _Lark(g)
  853. self.assertEqual(l.parse(u'AABB'), Tree('start', ['A', 'A', 'B', 'B']))
  854. self.assertEqual(l.parse(u'AAABB'), Tree('start', ['A', 'A', 'A', 'B', 'B']))
  855. self.assertRaises(ParseError, l.parse, u'AAAB')
  856. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAABBB')
  857. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB')
  858. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAABB')
  859. def test_ranged_repeat_terms(self):
  860. g = u"""!start: AAA
  861. AAA: "A"~3
  862. """
  863. l = _Lark(g)
  864. self.assertEqual(l.parse(u'AAA'), Tree('start', ["AAA"]))
  865. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AA')
  866. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAA')
  867. g = u"""!start: AABB CC
  868. AABB: "A"~0..2 "B"~2
  869. CC: "C"~1..2
  870. """
  871. l = _Lark(g)
  872. self.assertEqual(l.parse(u'AABBCC'), Tree('start', ['AABB', 'CC']))
  873. self.assertEqual(l.parse(u'BBC'), Tree('start', ['BB', 'C']))
  874. self.assertEqual(l.parse(u'ABBCC'), Tree('start', ['ABB', 'CC']))
  875. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAB')
  876. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAABBB')
  877. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB')
  878. self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAABB')
  879. _NAME = "Test" + PARSER.capitalize() + (LEXER or 'Scanless').capitalize()
  880. _TestParser.__name__ = _NAME
  881. globals()[_NAME] = _TestParser
  882. # Note: You still have to import them in __main__ for the tests to run
  883. _TO_TEST = [
  884. ('standard', 'earley'),
  885. ('standard', 'cyk'),
  886. ('dynamic', 'earley'),
  887. ('standard', 'lalr'),
  888. ('contextual', 'lalr'),
  889. (None, 'earley'),
  890. ]
  891. for _LEXER, _PARSER in _TO_TEST:
  892. _make_parser_test(_LEXER, _PARSER)
  893. for _LEXER in (None, 'dynamic'):
  894. _make_full_earley_test(_LEXER)
  895. if __name__ == '__main__':
  896. unittest.main()