From 7d21c754a12eccbe5cc842c5831bc2c3d7bae68f Mon Sep 17 00:00:00 2001 From: Kaspar Emanuel Date: Sun, 15 Oct 2017 18:27:55 +0100 Subject: [PATCH 1/4] Add test for UTF-8 characters in grammar --- tests/test_parser.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_parser.py b/tests/test_parser.py index 9fa05eb..d29aa47 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import unittest @@ -53,6 +54,19 @@ class TestParsers(unittest.TestCase): l = Lark(g, parser='earley', lexer='dynamic') self.assertRaises(ParseError, l.parse, 'a') + def test_utf8(self): + g = """start: a + a: "±a" + """ + l = Lark(g) + l.parse('±a') + + l = Lark(g, parser='earley', lexer=None) + l.parse('±a') + + l = Lark(g, parser='earley', lexer='dynamic') + l.parse('±a') + def _make_full_earley_test(LEXER): class _TestFullEarley(unittest.TestCase): From 9110e1e40046079c847737b87a0ea96f7373a37d Mon Sep 17 00:00:00 2001 From: Kaspar Emanuel Date: Sun, 15 Oct 2017 18:36:20 +0100 Subject: [PATCH 2/4] Fix installation of nearley deps for travis --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c539596..1a83b2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,4 +8,6 @@ python: - "3.6" - "pypy" # PyPy2 2.5.0 - "pypy3" # Pypy3 2.4.0 -script: python -m tests +script: + - pip install -r nearley-requirements.txt + - python -m tests From ed04b22c4c4aad2b5adb2a6dfecf9df3b5bbc93f Mon Sep 17 00:00:00 2001 From: Kaspar Emanuel Date: Sun, 15 Oct 2017 18:52:51 +0100 Subject: [PATCH 3/4] Fix UTF-8 test --- tests/test_parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_parser.py b/tests/test_parser.py index d29aa47..37729e1 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -55,17 +55,17 @@ class TestParsers(unittest.TestCase): self.assertRaises(ParseError, l.parse, 'a') def test_utf8(self): - g = """start: a + g = u"""start: a a: "±a" """ l = Lark(g) - l.parse('±a') + l.parse(u'±a') l = Lark(g, parser='earley', lexer=None) - l.parse('±a') + l.parse(u'±a') l = Lark(g, parser='earley', lexer='dynamic') - l.parse('±a') + l.parse(u'±a') def _make_full_earley_test(LEXER): From b532bf4e3c700f7af0ae2b65ca0c8f2d6bf625e6 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Mon, 16 Oct 2017 10:28:53 +0300 Subject: [PATCH 4/4] Fixed test --- tests/test_parser.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/test_parser.py b/tests/test_parser.py index 37729e1..b1a32c2 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -54,20 +54,6 @@ class TestParsers(unittest.TestCase): l = Lark(g, parser='earley', lexer='dynamic') self.assertRaises(ParseError, l.parse, 'a') - def test_utf8(self): - g = u"""start: a - a: "±a" - """ - l = Lark(g) - l.parse(u'±a') - - l = Lark(g, parser='earley', lexer=None) - l.parse(u'±a') - - l = Lark(g, parser='earley', lexer='dynamic') - l.parse(u'±a') - - def _make_full_earley_test(LEXER): class _TestFullEarley(unittest.TestCase): def test_anon_in_scanless(self): @@ -797,6 +783,22 @@ def _make_parser_test(LEXER, PARSER): self.assertEqual(''.join(child.data for child in res.children), 'indirection') + def test_utf8(self): + g = u"""start: a + a: "±a" + """ + l = _Lark(g) + self.assertEqual(l.parse(u'±a'), Tree('start', [Tree('a', [])])) + + g = u"""start: A + A: "±a" + """ + l = _Lark(g) + self.assertEqual(l.parse(u'±a'), Tree('start', [u'\xb1a'])) + + + + _NAME = "Test" + PARSER.capitalize() + (LEXER or 'Scanless').capitalize() _TestParser.__name__ = _NAME globals()[_NAME] = _TestParser