diff --git a/lark/__init__.py b/lark/__init__.py index 9bd88b0..1b5e7e3 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,4 +1,4 @@ -from .common import logger +from .utils import logger from .tree import Tree from .visitors import Transformer, Visitor, v_args, Discard from .visitors import InlineTransformer, inline_args # XXX Deprecated diff --git a/lark/common.py b/lark/common.py index b333dcb..714399a 100644 --- a/lark/common.py +++ b/lark/common.py @@ -1,13 +1,6 @@ -import logging from .utils import Serialize from .lexer import TerminalDef -logger = logging.getLogger("lark") -logger.addHandler(logging.StreamHandler()) -# Set to highest level, since we have some warnings amongst the code -# By default, we should not output any log messages -logger.setLevel(logging.CRITICAL) - ###{standalone class LexerConf(Serialize): diff --git a/lark/exceptions.py b/lark/exceptions.py index d1b956d..9d2d8dc 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -1,7 +1,6 @@ -from .utils import STRING_TYPE +from .utils import STRING_TYPE, logger ###{standalone -import logging class LarkError(Exception): @@ -62,24 +61,24 @@ class UnexpectedInput(LarkError): except UnexpectedInput as ut: if ut.state == self.state: if use_accepts and ut.accepts != self.accepts: - logging.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" % + logger.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" % (self.state, self.accepts, ut.accepts, i, j)) continue try: if ut.token == self.token: # Try exact match first - logging.debug("Exact Match at example [%s][%s]" % (i, j)) + logger.debug("Exact Match at example [%s][%s]" % (i, j)) return label if token_type_match_fallback: # Fallback to token types match if (ut.token.type == self.token.type) and not candidate[-1]: - logging.debug("Token Type Fallback at example [%s][%s]" % (i, j)) + logger.debug("Token Type Fallback at example [%s][%s]" % (i, j)) candidate = label, True except AttributeError: pass if not candidate[0]: - logging.debug("Same State match at example [%s][%s]" % (i, j)) + logger.debug("Same State match at example [%s][%s]" % (i, j)) candidate = label, False return candidate[0] diff --git a/lark/lark.py b/lark/lark.py index ddea2d6..9a4e001 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -4,10 +4,10 @@ import sys, os, pickle, hashlib from io import open -from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii +from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger from .load_grammar import load_grammar from .tree import Tree -from .common import LexerConf, ParserConf, logger +from .common import LexerConf, ParserConf from .lexer import Lexer, TraditionalLexer, TerminalDef, UnexpectedToken from .parse_tree_builder import ParseTreeBuilder diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index bf099e6..098639d 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -14,7 +14,7 @@ from collections import deque from ..visitors import Transformer_InPlace, v_args from ..exceptions import UnexpectedEOF, UnexpectedToken -from ..common import logger +from ..utils import logger from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal from .earley_common import Item, TransitiveItem diff --git a/lark/parsers/lalr_analysis.py b/lark/parsers/lalr_analysis.py index 861941f..7a94b4d 100644 --- a/lark/parsers/lalr_analysis.py +++ b/lark/parsers/lalr_analysis.py @@ -6,11 +6,10 @@ For now, shift/reduce conflicts are automatically resolved as shifts. # Author: Erez Shinan (2017) # Email : erezshin@gmail.com -from collections import defaultdict, deque +from collections import defaultdict -from ..utils import classify, classify_bool, bfs, fzset, Serialize, Enumerator +from ..utils import classify, classify_bool, bfs, fzset, Enumerator, logger from ..exceptions import GrammarError -from ..common import logger from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet from ..grammar import Rule diff --git a/lark/utils.py b/lark/utils.py index c70b947..0c41e6b 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -4,51 +4,15 @@ from functools import reduce from ast import literal_eval from collections import deque -class fzset(frozenset): - def __repr__(self): - return '{%s}' % ', '.join(map(repr, self)) - - -def classify_bool(seq, pred): - true_elems = [] - false_elems = [] - - for elem in seq: - if pred(elem): - true_elems.append(elem) - else: - false_elems.append(elem) - - return true_elems, false_elems - - - -def bfs(initial, expand): - open_q = deque(list(initial)) - visited = set(open_q) - while open_q: - node = open_q.popleft() - yield node - for next_node in expand(node): - if next_node not in visited: - visited.add(next_node) - open_q.append(next_node) - - +###{standalone +import logging +logger = logging.getLogger("lark") +logger.addHandler(logging.StreamHandler()) +# Set to highest level, since we have some warnings amongst the code +# By default, we should not output any log messages +logger.setLevel(logging.CRITICAL) -def _serialize(value, memo): - if isinstance(value, Serialize): - return value.serialize(memo) - elif isinstance(value, list): - return [_serialize(elem, memo) for elem in value] - elif isinstance(value, frozenset): - return list(value) # TODO reversible? - elif isinstance(value, dict): - return {key:_serialize(elem, memo) for key, elem in value.items()} - return value - -###{standalone def classify(seq, key=None, value=None): d = {} for item in seq: @@ -302,13 +266,11 @@ def combine_alternatives(lists): return reduce(lambda a,b: [i+[j] for i in a for j in b], lists[1:], init) - class FS: open = open exists = os.path.exists - def isascii(s): """ str.isascii only exists in python3.7+ """ try: @@ -318,4 +280,46 @@ def isascii(s): s.encode('ascii') return True except (UnicodeDecodeError, UnicodeEncodeError): - return False \ No newline at end of file + return False + + +class fzset(frozenset): + def __repr__(self): + return '{%s}' % ', '.join(map(repr, self)) + + +def classify_bool(seq, pred): + true_elems = [] + false_elems = [] + + for elem in seq: + if pred(elem): + true_elems.append(elem) + else: + false_elems.append(elem) + + return true_elems, false_elems + + +def bfs(initial, expand): + open_q = deque(list(initial)) + visited = set(open_q) + while open_q: + node = open_q.popleft() + yield node + for next_node in expand(node): + if next_node not in visited: + visited.add(next_node) + open_q.append(next_node) + + +def _serialize(value, memo): + if isinstance(value, Serialize): + return value.serialize(memo) + elif isinstance(value, list): + return [_serialize(elem, memo) for elem in value] + elif isinstance(value, frozenset): + return list(value) # TODO reversible? + elif isinstance(value, dict): + return {key:_serialize(elem, memo) for key, elem in value.items()} + return value \ No newline at end of file