From 209ac5ab4e893ef59a3bb4bdb5e01c612fa7ff64 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Thu, 14 Dec 2017 17:20:03 +0200 Subject: [PATCH] BUGFIX: Mishandling of quotes (Issue #50) --- lark/load_grammar.py | 2 +- tests/test_parser.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 092c118..72e2e22 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -293,7 +293,6 @@ def _rfind(s, choices): def _fix_escaping(s): - s = s.replace('\\"', '"').replace("'", "\\'") w = '' i = iter(s) for n in i: @@ -305,6 +304,7 @@ def _fix_escaping(s): elif n2 not in 'unftr': w += '\\' w += n2 + w = w.replace('\\"', '"').replace("'", "\\'") to_eval = "u'''%s'''" % w try: diff --git a/tests/test_parser.py b/tests/test_parser.py index 9ef7ab5..d93e33b 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -711,6 +711,19 @@ def _make_parser_test(LEXER, PARSER): """) x = g.parse('AB') + @unittest.skipIf(LEXER == None, "Scanless can't handle regexps") + def test_regex_quote(self): + g = r""" + start: SINGLE_QUOTED_STRING | DOUBLE_QUOTED_STRING + SINGLE_QUOTED_STRING : /'[^']*'/ + DOUBLE_QUOTED_STRING : /"[^"]*"/ + """ + + g = _Lark(g) + self.assertEqual( g.parse('"hello"').children, ['"hello"']) + self.assertEqual( g.parse("'hello'").children, ["'hello'"]) + + def test_lexer_token_limit(self): "Python has a stupid limit of 100 groups in a regular expression. Test that we handle this limitation" tokens = {'A%d'%i:'"%d"'%i for i in range(300)}