@@ -1,4 +1,4 @@ | |||
from .common import logger | |||
from .utils import logger | |||
from .tree import Tree | |||
from .visitors import Transformer, Visitor, v_args, Discard | |||
from .visitors import InlineTransformer, inline_args # XXX Deprecated | |||
@@ -1,13 +1,6 @@ | |||
import logging | |||
from .utils import Serialize | |||
from .lexer import TerminalDef | |||
logger = logging.getLogger("lark") | |||
logger.addHandler(logging.StreamHandler()) | |||
# Set to highest level, since we have some warnings amongst the code | |||
# By default, we should not output any log messages | |||
logger.setLevel(logging.CRITICAL) | |||
###{standalone | |||
class LexerConf(Serialize): | |||
@@ -1,7 +1,6 @@ | |||
from .utils import STRING_TYPE | |||
from .utils import STRING_TYPE, logger | |||
###{standalone | |||
import logging | |||
class LarkError(Exception): | |||
@@ -62,24 +61,24 @@ class UnexpectedInput(LarkError): | |||
except UnexpectedInput as ut: | |||
if ut.state == self.state: | |||
if use_accepts and ut.accepts != self.accepts: | |||
logging.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" % | |||
logger.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" % | |||
(self.state, self.accepts, ut.accepts, i, j)) | |||
continue | |||
try: | |||
if ut.token == self.token: # Try exact match first | |||
logging.debug("Exact Match at example [%s][%s]" % (i, j)) | |||
logger.debug("Exact Match at example [%s][%s]" % (i, j)) | |||
return label | |||
if token_type_match_fallback: | |||
# Fallback to token types match | |||
if (ut.token.type == self.token.type) and not candidate[-1]: | |||
logging.debug("Token Type Fallback at example [%s][%s]" % (i, j)) | |||
logger.debug("Token Type Fallback at example [%s][%s]" % (i, j)) | |||
candidate = label, True | |||
except AttributeError: | |||
pass | |||
if not candidate[0]: | |||
logging.debug("Same State match at example [%s][%s]" % (i, j)) | |||
logger.debug("Same State match at example [%s][%s]" % (i, j)) | |||
candidate = label, False | |||
return candidate[0] | |||
@@ -4,10 +4,10 @@ import sys, os, pickle, hashlib | |||
from io import open | |||
from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii | |||
from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger | |||
from .load_grammar import load_grammar | |||
from .tree import Tree | |||
from .common import LexerConf, ParserConf, logger | |||
from .common import LexerConf, ParserConf | |||
from .lexer import Lexer, TraditionalLexer, TerminalDef, UnexpectedToken | |||
from .parse_tree_builder import ParseTreeBuilder | |||
@@ -14,7 +14,7 @@ from collections import deque | |||
from ..visitors import Transformer_InPlace, v_args | |||
from ..exceptions import UnexpectedEOF, UnexpectedToken | |||
from ..common import logger | |||
from ..utils import logger | |||
from .grammar_analysis import GrammarAnalyzer | |||
from ..grammar import NonTerminal | |||
from .earley_common import Item, TransitiveItem | |||
@@ -6,11 +6,10 @@ For now, shift/reduce conflicts are automatically resolved as shifts. | |||
# Author: Erez Shinan (2017) | |||
# Email : erezshin@gmail.com | |||
from collections import defaultdict, deque | |||
from collections import defaultdict | |||
from ..utils import classify, classify_bool, bfs, fzset, Serialize, Enumerator | |||
from ..utils import classify, classify_bool, bfs, fzset, Enumerator, logger | |||
from ..exceptions import GrammarError | |||
from ..common import logger | |||
from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet | |||
from ..grammar import Rule | |||
@@ -4,51 +4,15 @@ from functools import reduce | |||
from ast import literal_eval | |||
from collections import deque | |||
class fzset(frozenset): | |||
def __repr__(self): | |||
return '{%s}' % ', '.join(map(repr, self)) | |||
def classify_bool(seq, pred): | |||
true_elems = [] | |||
false_elems = [] | |||
for elem in seq: | |||
if pred(elem): | |||
true_elems.append(elem) | |||
else: | |||
false_elems.append(elem) | |||
return true_elems, false_elems | |||
def bfs(initial, expand): | |||
open_q = deque(list(initial)) | |||
visited = set(open_q) | |||
while open_q: | |||
node = open_q.popleft() | |||
yield node | |||
for next_node in expand(node): | |||
if next_node not in visited: | |||
visited.add(next_node) | |||
open_q.append(next_node) | |||
###{standalone | |||
import logging | |||
logger = logging.getLogger("lark") | |||
logger.addHandler(logging.StreamHandler()) | |||
# Set to highest level, since we have some warnings amongst the code | |||
# By default, we should not output any log messages | |||
logger.setLevel(logging.CRITICAL) | |||
def _serialize(value, memo): | |||
if isinstance(value, Serialize): | |||
return value.serialize(memo) | |||
elif isinstance(value, list): | |||
return [_serialize(elem, memo) for elem in value] | |||
elif isinstance(value, frozenset): | |||
return list(value) # TODO reversible? | |||
elif isinstance(value, dict): | |||
return {key:_serialize(elem, memo) for key, elem in value.items()} | |||
return value | |||
###{standalone | |||
def classify(seq, key=None, value=None): | |||
d = {} | |||
for item in seq: | |||
@@ -302,13 +266,11 @@ def combine_alternatives(lists): | |||
return reduce(lambda a,b: [i+[j] for i in a for j in b], lists[1:], init) | |||
class FS: | |||
open = open | |||
exists = os.path.exists | |||
def isascii(s): | |||
""" str.isascii only exists in python3.7+ """ | |||
try: | |||
@@ -318,4 +280,46 @@ def isascii(s): | |||
s.encode('ascii') | |||
return True | |||
except (UnicodeDecodeError, UnicodeEncodeError): | |||
return False | |||
return False | |||
class fzset(frozenset): | |||
def __repr__(self): | |||
return '{%s}' % ', '.join(map(repr, self)) | |||
def classify_bool(seq, pred): | |||
true_elems = [] | |||
false_elems = [] | |||
for elem in seq: | |||
if pred(elem): | |||
true_elems.append(elem) | |||
else: | |||
false_elems.append(elem) | |||
return true_elems, false_elems | |||
def bfs(initial, expand): | |||
open_q = deque(list(initial)) | |||
visited = set(open_q) | |||
while open_q: | |||
node = open_q.popleft() | |||
yield node | |||
for next_node in expand(node): | |||
if next_node not in visited: | |||
visited.add(next_node) | |||
open_q.append(next_node) | |||
def _serialize(value, memo): | |||
if isinstance(value, Serialize): | |||
return value.serialize(memo) | |||
elif isinstance(value, list): | |||
return [_serialize(elem, memo) for elem in value] | |||
elif isinstance(value, frozenset): | |||
return list(value) # TODO reversible? | |||
elif isinstance(value, dict): | |||
return {key:_serialize(elem, memo) for key, elem in value.items()} | |||
return value |