From 5c8a25c7333ea685d5816dea186f0cf389d6d7f5 Mon Sep 17 00:00:00 2001 From: pwwang Date: Tue, 30 Jun 2020 18:18:49 -0500 Subject: [PATCH 1/4] Avoid using root logger --- docs/how_to_use.md | 7 ++-- lark/__init__.py | 1 + lark/common.py | 7 ++++ lark/lark.py | 8 ++-- lark/parsers/earley.py | 4 +- lark/parsers/lalr_analysis.py | 6 +-- tests/__main__.py | 7 +++- tests/test_logger.py | 65 ++++++++++++++++++++++++++++++ tests/test_nearley/test_nearley.py | 7 ++-- tests/test_parser.py | 3 +- 10 files changed, 97 insertions(+), 18 deletions(-) create mode 100644 tests/test_logger.py diff --git a/docs/how_to_use.md b/docs/how_to_use.md index 886b440..78f4df2 100644 --- a/docs/how_to_use.md +++ b/docs/how_to_use.md @@ -30,12 +30,13 @@ Use the reference pages for more in-depth explanations. (links in the [main page ## LALR usage -By default Lark silently resolves Shift/Reduce conflicts as Shift. To enable warnings pass `debug=True`. To get the messages printed you have to configure `logging` framework beforehand. For example: +By default Lark silently resolves Shift/Reduce conflicts as Shift. To enable warnings pass `debug=True`. To get the messages printed you have to configure the `LOGGER` beforehand. For example: ```python -from lark import Lark import logging -logging.basicConfig(level=logging.DEBUG) +from lark import Lark, LOGGER + +LOGGER.setLevel(logging.DEBUG) collision_grammar = ''' start: as as diff --git a/lark/__init__.py b/lark/__init__.py index 9e50691..e4c54dd 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,3 +1,4 @@ +from .common import LOGGER from .tree import Tree from .visitors import Transformer, Visitor, v_args, Discard from .visitors import InlineTransformer, inline_args # XXX Deprecated diff --git a/lark/common.py b/lark/common.py index c44f9ce..aac9d75 100644 --- a/lark/common.py +++ b/lark/common.py @@ -1,6 +1,13 @@ +import logging from .utils import Serialize from .lexer import TerminalDef +LOGGER = logging.getLogger("LARK") +LOGGER.addHandler(logging.StreamHandler()) +# Set to highest level, since we have some warnings amongst the code +# By default, we should not output any log messages +LOGGER.setLevel(logging.CRITICAL) + ###{standalone class LexerConf(Serialize): diff --git a/lark/lark.py b/lark/lark.py index 2b783cb..8df2b87 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -1,13 +1,13 @@ from __future__ import absolute_import -import sys, os, pickle, hashlib, logging +import sys, os, pickle, hashlib from io import open from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS from .load_grammar import load_grammar from .tree import Tree -from .common import LexerConf, ParserConf +from .common import LexerConf, ParserConf, LOGGER from .lexer import Lexer, TraditionalLexer, TerminalDef, UnexpectedToken from .parse_tree_builder import ParseTreeBuilder @@ -205,7 +205,7 @@ class Lark(Serialize): cache_fn = '.lark_cache_%s.tmp' % md5 if FS.exists(cache_fn): - logging.debug('Loading grammar from cache: %s', cache_fn) + LOGGER.debug('Loading grammar from cache: %s', cache_fn) with FS.open(cache_fn, 'rb') as f: self._load(f, self.options.transformer, self.options.postlex) return @@ -284,7 +284,7 @@ class Lark(Serialize): self.lexer = self._build_lexer() if cache_fn: - logging.debug('Saving grammar to cache: %s', cache_fn) + LOGGER.debug('Saving grammar to cache: %s', cache_fn) with FS.open(cache_fn, 'wb') as f: self.save(f) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 59e9a06..5fc7531 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -10,11 +10,11 @@ is better documented here: http://www.bramvandersanden.com/post/2014/06/shared-packed-parse-forest/ """ -import logging from collections import deque from ..visitors import Transformer_InPlace, v_args from ..exceptions import UnexpectedEOF, UnexpectedToken +from ..common import LOGGER from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal from .earley_common import Item, TransitiveItem @@ -301,7 +301,7 @@ class Parser: try: debug_walker = ForestToPyDotVisitor() except ImportError: - logging.warning("Cannot find dependency 'pydot', will not generate sppf debug image") + LOGGER.warning("Cannot find dependency 'pydot', will not generate sppf debug image") else: debug_walker.visit(solutions[0], "sppf.png") diff --git a/lark/parsers/lalr_analysis.py b/lark/parsers/lalr_analysis.py index 8890c3c..6fefa4c 100644 --- a/lark/parsers/lalr_analysis.py +++ b/lark/parsers/lalr_analysis.py @@ -6,11 +6,11 @@ For now, shift/reduce conflicts are automatically resolved as shifts. # Author: Erez Shinan (2017) # Email : erezshin@gmail.com -import logging from collections import defaultdict, deque from ..utils import classify, classify_bool, bfs, fzset, Serialize, Enumerator from ..exceptions import GrammarError +from ..common import LOGGER from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet from ..grammar import Rule @@ -256,8 +256,8 @@ class LALR_Analyzer(GrammarAnalyzer): raise GrammarError('Reduce/Reduce collision in %s between the following rules: %s' % (la, ''.join([ '\n\t\t- ' + str(r) for r in rules ]))) if la in actions: if self.debug: - logging.warning('Shift/Reduce conflict for terminal %s: (resolving as shift)', la.name) - logging.warning(' * %s', list(rules)[0]) + LOGGER.warning('Shift/Reduce conflict for terminal %s: (resolving as shift)', la.name) + LOGGER.warning(' * %s', list(rules)[0]) else: actions[la] = (Reduce, list(rules)[0]) m[state] = { k.name: v for k, v in actions.items() } diff --git a/tests/__main__.py b/tests/__main__.py index cb26eb4..1807aa8 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function import unittest import logging +from lark import LOGGER from .test_trees import TestTrees from .test_tools import TestStandalone @@ -11,11 +12,13 @@ from .test_reconstructor import TestReconstructor try: from .test_nearley.test_nearley import TestNearley except ImportError: - logging.warning("Warning: Skipping tests for Nearley grammar imports (js2py required)") + LOGGER.warning("Warning: Skipping tests for Nearley grammar imports (js2py required)") # from .test_selectors import TestSelectors # from .test_grammars import TestPythonG, TestConfigG +from .test_logger import TestLogger + from .test_parser import ( TestLalrStandard, TestEarleyStandard, @@ -31,7 +34,7 @@ from .test_parser import ( TestParsers, ) -logging.basicConfig(level=logging.INFO) +LOGGER.setLevel(logging.INFO) if __name__ == '__main__': unittest.main() diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 0000000..dd6beb3 --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,65 @@ +import logging +from contextlib import contextmanager +from lark import Lark, LOGGER +from unittest import TestCase, main + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +@contextmanager +def capture_log(): + stream = StringIO() + orig_handler = LOGGER.handlers[0] + del LOGGER.handlers[:] + LOGGER.addHandler(logging.StreamHandler(stream)) + yield stream + del LOGGER.handlers[:] + LOGGER.addHandler(orig_handler) + +class TestLogger(TestCase): + + def test_debug(self): + LOGGER.setLevel(logging.DEBUG) + collision_grammar = ''' + start: as as + as: a* + a: "a" + ''' + with capture_log() as log: + Lark(collision_grammar, parser='lalr', debug=True) + + log = log.getvalue() + self.assertIn("Shift/Reduce conflict for terminal", log) + self.assertIn("A: (resolving as shift)", log) + self.assertIn("Shift/Reduce conflict for terminal A: (resolving as shift)", log) + + def test_non_debug(self): + LOGGER.setLevel(logging.DEBUG) + collision_grammar = ''' + start: as as + as: a* + a: "a" + ''' + with capture_log() as log: + Lark(collision_grammar, parser='lalr', debug=False) + log = log.getvalue() + # no log messge + self.assertEqual(len(log), 0) + + def test_loglevel_higher(self): + LOGGER.setLevel(logging.ERROR) + collision_grammar = ''' + start: as as + as: a* + a: "a" + ''' + with capture_log() as log: + Lark(collision_grammar, parser='lalr', debug=True) + log = log.getvalue() + # no log messge + self.assertEqual(len(log), 0) + +if __name__ == '__main__': + main() diff --git a/tests/test_nearley/test_nearley.py b/tests/test_nearley/test_nearley.py index 647f489..345af8a 100644 --- a/tests/test_nearley/test_nearley.py +++ b/tests/test_nearley/test_nearley.py @@ -6,16 +6,17 @@ import logging import os import codecs -logging.basicConfig(level=logging.INFO) - +from lark import LOGGER from lark.tools.nearley import create_code_for_nearley_grammar, main as nearley_tool_main +LOGGER.setLevel(logging.INFO) + TEST_PATH = os.path.abspath(os.path.dirname(__file__)) NEARLEY_PATH = os.path.join(TEST_PATH, 'nearley') BUILTIN_PATH = os.path.join(NEARLEY_PATH, 'builtin') if not os.path.exists(NEARLEY_PATH): - logging.warn("Nearley not installed. Skipping Nearley tests!") + LOGGER.warn("Nearley not installed. Skipping Nearley tests!") raise ImportError("Skipping Nearley tests!") import js2py # Ensures that js2py exists, to avoid failing tests diff --git a/tests/test_parser.py b/tests/test_parser.py index df09307..5a10b9f 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -18,13 +18,13 @@ from io import ( open, ) -logging.basicConfig(level=logging.INFO) try: import regex except ImportError: regex = None +from lark import LOGGER from lark.lark import Lark from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters from lark.tree import Tree @@ -32,6 +32,7 @@ from lark.visitors import Transformer, Transformer_InPlace, v_args from lark.grammar import Rule from lark.lexer import TerminalDef, Lexer, TraditionalLexer +LOGGER.setLevel(logging.INFO) __path__ = os.path.dirname(__file__) From a6201b41e471897ef044696925911df86b94a886 Mon Sep 17 00:00:00 2001 From: pwwang <1188067+pwwang@users.noreply.github.com> Date: Tue, 30 Jun 2020 17:35:26 -0700 Subject: [PATCH 2/4] Lowercase logger name --- lark/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lark/common.py b/lark/common.py index aac9d75..3bd7c98 100644 --- a/lark/common.py +++ b/lark/common.py @@ -2,7 +2,7 @@ import logging from .utils import Serialize from .lexer import TerminalDef -LOGGER = logging.getLogger("LARK") +LOGGER = logging.getLogger("lark") LOGGER.addHandler(logging.StreamHandler()) # Set to highest level, since we have some warnings amongst the code # By default, we should not output any log messages From 2a73afd3554c29f216869bc3e70f971f74b62c13 Mon Sep 17 00:00:00 2001 From: pwwang Date: Thu, 2 Jul 2020 19:28:45 -0500 Subject: [PATCH 3/4] Change LOGGER to logger --- docs/how_to_use.md | 6 +++--- lark/__init__.py | 2 +- lark/common.py | 6 +++--- lark/lark.py | 6 +++--- lark/parsers/earley.py | 4 ++-- lark/parsers/lalr_analysis.py | 6 +++--- tests/__main__.py | 8 ++++---- tests/test_logger.py | 26 +++++++++++++------------- tests/test_nearley/test_nearley.py | 6 +++--- tests/test_parser.py | 4 ++-- 10 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/how_to_use.md b/docs/how_to_use.md index 78f4df2..303098f 100644 --- a/docs/how_to_use.md +++ b/docs/how_to_use.md @@ -30,13 +30,13 @@ Use the reference pages for more in-depth explanations. (links in the [main page ## LALR usage -By default Lark silently resolves Shift/Reduce conflicts as Shift. To enable warnings pass `debug=True`. To get the messages printed you have to configure the `LOGGER` beforehand. For example: +By default Lark silently resolves Shift/Reduce conflicts as Shift. To enable warnings pass `debug=True`. To get the messages printed you have to configure the `logger` beforehand. For example: ```python import logging -from lark import Lark, LOGGER +from lark import Lark, logger -LOGGER.setLevel(logging.DEBUG) +logger.setLevel(logging.DEBUG) collision_grammar = ''' start: as as diff --git a/lark/__init__.py b/lark/__init__.py index e4c54dd..e3021cf 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,4 +1,4 @@ -from .common import LOGGER +from .common import logger from .tree import Tree from .visitors import Transformer, Visitor, v_args, Discard from .visitors import InlineTransformer, inline_args # XXX Deprecated diff --git a/lark/common.py b/lark/common.py index 3bd7c98..745e287 100644 --- a/lark/common.py +++ b/lark/common.py @@ -2,11 +2,11 @@ import logging from .utils import Serialize from .lexer import TerminalDef -LOGGER = logging.getLogger("lark") -LOGGER.addHandler(logging.StreamHandler()) +logger = logging.getLogger("lark") +logger.addHandler(logging.StreamHandler()) # Set to highest level, since we have some warnings amongst the code # By default, we should not output any log messages -LOGGER.setLevel(logging.CRITICAL) +logger.setLevel(logging.CRITICAL) ###{standalone diff --git a/lark/lark.py b/lark/lark.py index 8df2b87..9bb60c8 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -7,7 +7,7 @@ from io import open from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS from .load_grammar import load_grammar from .tree import Tree -from .common import LexerConf, ParserConf, LOGGER +from .common import LexerConf, ParserConf, logger from .lexer import Lexer, TraditionalLexer, TerminalDef, UnexpectedToken from .parse_tree_builder import ParseTreeBuilder @@ -205,7 +205,7 @@ class Lark(Serialize): cache_fn = '.lark_cache_%s.tmp' % md5 if FS.exists(cache_fn): - LOGGER.debug('Loading grammar from cache: %s', cache_fn) + logger.debug('Loading grammar from cache: %s', cache_fn) with FS.open(cache_fn, 'rb') as f: self._load(f, self.options.transformer, self.options.postlex) return @@ -284,7 +284,7 @@ class Lark(Serialize): self.lexer = self._build_lexer() if cache_fn: - LOGGER.debug('Saving grammar to cache: %s', cache_fn) + logger.debug('Saving grammar to cache: %s', cache_fn) with FS.open(cache_fn, 'wb') as f: self.save(f) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 5fc7531..bf099e6 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,7 @@ from collections import deque from ..visitors import Transformer_InPlace, v_args from ..exceptions import UnexpectedEOF, UnexpectedToken -from ..common import LOGGER +from ..common import logger from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal from .earley_common import Item, TransitiveItem @@ -301,7 +301,7 @@ class Parser: try: debug_walker = ForestToPyDotVisitor() except ImportError: - LOGGER.warning("Cannot find dependency 'pydot', will not generate sppf debug image") + logger.warning("Cannot find dependency 'pydot', will not generate sppf debug image") else: debug_walker.visit(solutions[0], "sppf.png") diff --git a/lark/parsers/lalr_analysis.py b/lark/parsers/lalr_analysis.py index 6fefa4c..861941f 100644 --- a/lark/parsers/lalr_analysis.py +++ b/lark/parsers/lalr_analysis.py @@ -10,7 +10,7 @@ from collections import defaultdict, deque from ..utils import classify, classify_bool, bfs, fzset, Serialize, Enumerator from ..exceptions import GrammarError -from ..common import LOGGER +from ..common import logger from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet from ..grammar import Rule @@ -256,8 +256,8 @@ class LALR_Analyzer(GrammarAnalyzer): raise GrammarError('Reduce/Reduce collision in %s between the following rules: %s' % (la, ''.join([ '\n\t\t- ' + str(r) for r in rules ]))) if la in actions: if self.debug: - LOGGER.warning('Shift/Reduce conflict for terminal %s: (resolving as shift)', la.name) - LOGGER.warning(' * %s', list(rules)[0]) + logger.warning('Shift/Reduce conflict for terminal %s: (resolving as shift)', la.name) + logger.warning(' * %s', list(rules)[0]) else: actions[la] = (Reduce, list(rules)[0]) m[state] = { k.name: v for k, v in actions.items() } diff --git a/tests/__main__.py b/tests/__main__.py index 1807aa8..9ef9f1b 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, print_function import unittest import logging -from lark import LOGGER +from lark import logger from .test_trees import TestTrees from .test_tools import TestStandalone @@ -12,12 +12,12 @@ from .test_reconstructor import TestReconstructor try: from .test_nearley.test_nearley import TestNearley except ImportError: - LOGGER.warning("Warning: Skipping tests for Nearley grammar imports (js2py required)") + logger.warning("Warning: Skipping tests for Nearley grammar imports (js2py required)") # from .test_selectors import TestSelectors # from .test_grammars import TestPythonG, TestConfigG -from .test_logger import TestLogger +from .test_logger import Testlogger from .test_parser import ( TestLalrStandard, @@ -34,7 +34,7 @@ from .test_parser import ( TestParsers, ) -LOGGER.setLevel(logging.INFO) +logger.setLevel(logging.INFO) if __name__ == '__main__': unittest.main() diff --git a/tests/test_logger.py b/tests/test_logger.py index dd6beb3..93dc8ed 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,6 +1,6 @@ import logging from contextlib import contextmanager -from lark import Lark, LOGGER +from lark import Lark, logger from unittest import TestCase, main try: @@ -11,17 +11,17 @@ except ImportError: @contextmanager def capture_log(): stream = StringIO() - orig_handler = LOGGER.handlers[0] - del LOGGER.handlers[:] - LOGGER.addHandler(logging.StreamHandler(stream)) + orig_handler = logger.handlers[0] + del logger.handlers[:] + logger.addHandler(logging.StreamHandler(stream)) yield stream - del LOGGER.handlers[:] - LOGGER.addHandler(orig_handler) + del logger.handlers[:] + logger.addHandler(orig_handler) -class TestLogger(TestCase): +class Testlogger(TestCase): def test_debug(self): - LOGGER.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) collision_grammar = ''' start: as as as: a* @@ -31,12 +31,12 @@ class TestLogger(TestCase): Lark(collision_grammar, parser='lalr', debug=True) log = log.getvalue() - self.assertIn("Shift/Reduce conflict for terminal", log) - self.assertIn("A: (resolving as shift)", log) - self.assertIn("Shift/Reduce conflict for terminal A: (resolving as shift)", log) + # since there are conflicts about A + # symbol A should appear in the log message for hint + self.assertIn("A", log) def test_non_debug(self): - LOGGER.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) collision_grammar = ''' start: as as as: a* @@ -49,7 +49,7 @@ class TestLogger(TestCase): self.assertEqual(len(log), 0) def test_loglevel_higher(self): - LOGGER.setLevel(logging.ERROR) + logger.setLevel(logging.ERROR) collision_grammar = ''' start: as as as: a* diff --git a/tests/test_nearley/test_nearley.py b/tests/test_nearley/test_nearley.py index 345af8a..1ad6449 100644 --- a/tests/test_nearley/test_nearley.py +++ b/tests/test_nearley/test_nearley.py @@ -6,17 +6,17 @@ import logging import os import codecs -from lark import LOGGER +from lark import logger from lark.tools.nearley import create_code_for_nearley_grammar, main as nearley_tool_main -LOGGER.setLevel(logging.INFO) +logger.setLevel(logging.INFO) TEST_PATH = os.path.abspath(os.path.dirname(__file__)) NEARLEY_PATH = os.path.join(TEST_PATH, 'nearley') BUILTIN_PATH = os.path.join(NEARLEY_PATH, 'builtin') if not os.path.exists(NEARLEY_PATH): - LOGGER.warn("Nearley not installed. Skipping Nearley tests!") + logger.warn("Nearley not installed. Skipping Nearley tests!") raise ImportError("Skipping Nearley tests!") import js2py # Ensures that js2py exists, to avoid failing tests diff --git a/tests/test_parser.py b/tests/test_parser.py index 5a10b9f..88d175f 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -24,7 +24,7 @@ try: except ImportError: regex = None -from lark import LOGGER +from lark import logger from lark.lark import Lark from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters from lark.tree import Tree @@ -32,7 +32,7 @@ from lark.visitors import Transformer, Transformer_InPlace, v_args from lark.grammar import Rule from lark.lexer import TerminalDef, Lexer, TraditionalLexer -LOGGER.setLevel(logging.INFO) +logger.setLevel(logging.INFO) __path__ = os.path.dirname(__file__) From af3bedd39d1f452a14d60afc528ee78490664dbf Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Fri, 14 Aug 2020 16:17:26 +0300 Subject: [PATCH 4/4] Adjustments to logging PR --- lark/__init__.py | 2 +- lark/common.py | 7 --- lark/exceptions.py | 11 ++-- lark/lark.py | 4 +- lark/parsers/earley.py | 2 +- lark/parsers/lalr_analysis.py | 5 +- lark/utils.py | 96 ++++++++++++++++++----------------- 7 files changed, 61 insertions(+), 66 deletions(-) diff --git a/lark/__init__.py b/lark/__init__.py index 9bd88b0..1b5e7e3 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,4 +1,4 @@ -from .common import logger +from .utils import logger from .tree import Tree from .visitors import Transformer, Visitor, v_args, Discard from .visitors import InlineTransformer, inline_args # XXX Deprecated diff --git a/lark/common.py b/lark/common.py index b333dcb..714399a 100644 --- a/lark/common.py +++ b/lark/common.py @@ -1,13 +1,6 @@ -import logging from .utils import Serialize from .lexer import TerminalDef -logger = logging.getLogger("lark") -logger.addHandler(logging.StreamHandler()) -# Set to highest level, since we have some warnings amongst the code -# By default, we should not output any log messages -logger.setLevel(logging.CRITICAL) - ###{standalone class LexerConf(Serialize): diff --git a/lark/exceptions.py b/lark/exceptions.py index d1b956d..9d2d8dc 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -1,7 +1,6 @@ -from .utils import STRING_TYPE +from .utils import STRING_TYPE, logger ###{standalone -import logging class LarkError(Exception): @@ -62,24 +61,24 @@ class UnexpectedInput(LarkError): except UnexpectedInput as ut: if ut.state == self.state: if use_accepts and ut.accepts != self.accepts: - logging.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" % + logger.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" % (self.state, self.accepts, ut.accepts, i, j)) continue try: if ut.token == self.token: # Try exact match first - logging.debug("Exact Match at example [%s][%s]" % (i, j)) + logger.debug("Exact Match at example [%s][%s]" % (i, j)) return label if token_type_match_fallback: # Fallback to token types match if (ut.token.type == self.token.type) and not candidate[-1]: - logging.debug("Token Type Fallback at example [%s][%s]" % (i, j)) + logger.debug("Token Type Fallback at example [%s][%s]" % (i, j)) candidate = label, True except AttributeError: pass if not candidate[0]: - logging.debug("Same State match at example [%s][%s]" % (i, j)) + logger.debug("Same State match at example [%s][%s]" % (i, j)) candidate = label, False return candidate[0] diff --git a/lark/lark.py b/lark/lark.py index ddea2d6..9a4e001 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -4,10 +4,10 @@ import sys, os, pickle, hashlib from io import open -from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii +from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger from .load_grammar import load_grammar from .tree import Tree -from .common import LexerConf, ParserConf, logger +from .common import LexerConf, ParserConf from .lexer import Lexer, TraditionalLexer, TerminalDef, UnexpectedToken from .parse_tree_builder import ParseTreeBuilder diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index bf099e6..098639d 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,7 @@ from collections import deque from ..visitors import Transformer_InPlace, v_args from ..exceptions import UnexpectedEOF, UnexpectedToken -from ..common import logger +from ..utils import logger from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal from .earley_common import Item, TransitiveItem diff --git a/lark/parsers/lalr_analysis.py b/lark/parsers/lalr_analysis.py index 861941f..7a94b4d 100644 --- a/lark/parsers/lalr_analysis.py +++ b/lark/parsers/lalr_analysis.py @@ -6,11 +6,10 @@ For now, shift/reduce conflicts are automatically resolved as shifts. # Author: Erez Shinan (2017) # Email : erezshin@gmail.com -from collections import defaultdict, deque +from collections import defaultdict -from ..utils import classify, classify_bool, bfs, fzset, Serialize, Enumerator +from ..utils import classify, classify_bool, bfs, fzset, Enumerator, logger from ..exceptions import GrammarError -from ..common import logger from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet from ..grammar import Rule diff --git a/lark/utils.py b/lark/utils.py index c70b947..0c41e6b 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -4,51 +4,15 @@ from functools import reduce from ast import literal_eval from collections import deque -class fzset(frozenset): - def __repr__(self): - return '{%s}' % ', '.join(map(repr, self)) - - -def classify_bool(seq, pred): - true_elems = [] - false_elems = [] - - for elem in seq: - if pred(elem): - true_elems.append(elem) - else: - false_elems.append(elem) - - return true_elems, false_elems - - - -def bfs(initial, expand): - open_q = deque(list(initial)) - visited = set(open_q) - while open_q: - node = open_q.popleft() - yield node - for next_node in expand(node): - if next_node not in visited: - visited.add(next_node) - open_q.append(next_node) - - +###{standalone +import logging +logger = logging.getLogger("lark") +logger.addHandler(logging.StreamHandler()) +# Set to highest level, since we have some warnings amongst the code +# By default, we should not output any log messages +logger.setLevel(logging.CRITICAL) -def _serialize(value, memo): - if isinstance(value, Serialize): - return value.serialize(memo) - elif isinstance(value, list): - return [_serialize(elem, memo) for elem in value] - elif isinstance(value, frozenset): - return list(value) # TODO reversible? - elif isinstance(value, dict): - return {key:_serialize(elem, memo) for key, elem in value.items()} - return value - -###{standalone def classify(seq, key=None, value=None): d = {} for item in seq: @@ -302,13 +266,11 @@ def combine_alternatives(lists): return reduce(lambda a,b: [i+[j] for i in a for j in b], lists[1:], init) - class FS: open = open exists = os.path.exists - def isascii(s): """ str.isascii only exists in python3.7+ """ try: @@ -318,4 +280,46 @@ def isascii(s): s.encode('ascii') return True except (UnicodeDecodeError, UnicodeEncodeError): - return False \ No newline at end of file + return False + + +class fzset(frozenset): + def __repr__(self): + return '{%s}' % ', '.join(map(repr, self)) + + +def classify_bool(seq, pred): + true_elems = [] + false_elems = [] + + for elem in seq: + if pred(elem): + true_elems.append(elem) + else: + false_elems.append(elem) + + return true_elems, false_elems + + +def bfs(initial, expand): + open_q = deque(list(initial)) + visited = set(open_q) + while open_q: + node = open_q.popleft() + yield node + for next_node in expand(node): + if next_node not in visited: + visited.add(next_node) + open_q.append(next_node) + + +def _serialize(value, memo): + if isinstance(value, Serialize): + return value.serialize(memo) + elif isinstance(value, list): + return [_serialize(elem, memo) for elem in value] + elif isinstance(value, frozenset): + return list(value) # TODO reversible? + elif isinstance(value, dict): + return {key:_serialize(elem, memo) for key, elem in value.items()} + return value \ No newline at end of file