From 926c98bd41f70824ecdb2c8086cdd17d74ab97f7 Mon Sep 17 00:00:00 2001 From: MegaIng1 Date: Fri, 27 Mar 2020 12:40:47 +0100 Subject: [PATCH] Renamed global_flags to g_regex_flags --- lark/common.py | 6 +++--- lark/lark.py | 8 ++++---- lark/lexer.py | 34 +++++++++++++++++----------------- lark/parser_frontends.py | 6 +++--- lark_stubs/lark.pyi | 4 ++-- lark_stubs/lexer.pyi | 4 ++-- tests/test_parser.py | 6 +++--- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/lark/common.py b/lark/common.py index 6c04810..c44f9ce 100644 --- a/lark/common.py +++ b/lark/common.py @@ -4,15 +4,15 @@ from .lexer import TerminalDef ###{standalone class LexerConf(Serialize): - __serialize_fields__ = 'tokens', 'ignore', 'global_flags' + __serialize_fields__ = 'tokens', 'ignore', 'g_regex_flags' __serialize_namespace__ = TerminalDef, - def __init__(self, tokens, ignore=(), postlex=None, callbacks=None, global_flags=0): + def __init__(self, tokens, ignore=(), postlex=None, callbacks=None, g_regex_flags=0): self.tokens = tokens self.ignore = ignore self.postlex = postlex self.callbacks = callbacks or {} - self.global_flags = global_flags + self.g_regex_flags = g_regex_flags def _deserialize(self): self.callbacks = {} # TODO diff --git a/lark/lark.py b/lark/lark.py index b871c7a..8710757 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -48,7 +48,7 @@ class LarkOptions(Serialize): propagate_positions - Propagates [line, column, end_line, end_column] attributes into all tree branches. lexer_callbacks - Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution. maybe_placeholders - Experimental feature. Instead of omitting optional rules (i.e. rule?), replace them with None - global_flags - Flags that are applied to all Terminals (Regex and Strings) + g_regex_flags - Flags that are applied to all Terminals (Regex and Strings) """ if __doc__: __doc__ += OPTIONS_DOC @@ -69,7 +69,7 @@ class LarkOptions(Serialize): 'lexer_callbacks': {}, 'maybe_placeholders': False, 'edit_terminals': None, - 'global_flags': 0, + 'g_regex_flags': 0, } def __init__(self, options_dict): @@ -211,7 +211,7 @@ class Lark(Serialize): if hasattr(t, term.name): lexer_callbacks[term.name] = getattr(t, term.name) - self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, lexer_callbacks, self.options.global_flags) + self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, lexer_callbacks, self.options.g_regex_flags) if self.options.parser: self.parser = self._build_parser() @@ -224,7 +224,7 @@ class Lark(Serialize): __serialize_fields__ = 'parser', 'rules', 'options' def _build_lexer(self): - return TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks, global_flags=self.lexer_conf.global_flags) + return TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks, g_regex_flags=self.lexer_conf.g_regex_flags) def _prepare_callbacks(self): self.parser_class = get_frontend(self.options.parser, self.options.lexer) diff --git a/lark/lexer.py b/lark/lexer.py index 2487317..32bfe78 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -230,7 +230,7 @@ class CallChain: -def _create_unless(terminals, global_flags): +def _create_unless(terminals, g_regex_flags): tokens_by_type = classify(terminals, lambda t: type(t.pattern)) assert len(tokens_by_type) <= 2, tokens_by_type.keys() embedded_strs = set() @@ -241,19 +241,19 @@ def _create_unless(terminals, global_flags): if strtok.priority > retok.priority: continue s = strtok.pattern.value - m = re.match(retok.pattern.to_regexp(), s, global_flags) + m = re.match(retok.pattern.to_regexp(), s, g_regex_flags) if m and m.group(0) == s: unless.append(strtok) if strtok.pattern.flags <= retok.pattern.flags: embedded_strs.add(strtok) if unless: - callback[retok.name] = UnlessCallback(build_mres(unless, global_flags, match_whole=True)) + callback[retok.name] = UnlessCallback(build_mres(unless, g_regex_flags, match_whole=True)) terminals = [t for t in terminals if t not in embedded_strs] return terminals, callback -def _build_mres(terminals, max_size, global_flags, match_whole): +def _build_mres(terminals, max_size, g_regex_flags, match_whole): # Python sets an unreasonable group limit (currently 100) in its re module # Worse, the only way to know we reached it is by catching an AssertionError! # This function recursively tries less and less groups until it's successful. @@ -261,17 +261,17 @@ def _build_mres(terminals, max_size, global_flags, match_whole): mres = [] while terminals: try: - mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in terminals[:max_size]), global_flags) + mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in terminals[:max_size]), g_regex_flags) except AssertionError: # Yes, this is what Python provides us.. :/ - return _build_mres(terminals, max_size//2, global_flags, match_whole) + return _build_mres(terminals, max_size//2, g_regex_flags, match_whole) # terms_from_name = {t.name: t for t in terminals[:max_size]} mres.append((mre, {i:n for n,i in mre.groupindex.items()} )) terminals = terminals[max_size:] return mres -def build_mres(terminals, global_flags, match_whole=False): - return _build_mres(terminals, len(terminals), global_flags, match_whole) +def build_mres(terminals, g_regex_flags, match_whole=False): + return _build_mres(terminals, len(terminals), g_regex_flags, match_whole) def _regexp_has_newline(r): r"""Expressions that may indicate newlines in a regexp: @@ -294,7 +294,7 @@ class Lexer(object): class TraditionalLexer(Lexer): - def __init__(self, terminals, ignore=(), user_callbacks={}, global_flags=0): + def __init__(self, terminals, ignore=(), user_callbacks={}, g_regex_flags=0): assert all(isinstance(t, TerminalDef) for t in terminals), terminals terminals = list(terminals) @@ -302,7 +302,7 @@ class TraditionalLexer(Lexer): # Sanitization for t in terminals: try: - re.compile(t.pattern.to_regexp(), global_flags) + re.compile(t.pattern.to_regexp(), g_regex_flags) except re.error: raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern)) @@ -318,10 +318,10 @@ class TraditionalLexer(Lexer): terminals.sort(key=lambda x:(-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name)) self.terminals = terminals self.user_callbacks = user_callbacks - self.build(global_flags) + self.build(g_regex_flags) - def build(self, global_flags=0): - terminals, self.callback = _create_unless(self.terminals, global_flags) + def build(self, g_regex_flags=0): + terminals, self.callback = _create_unless(self.terminals, g_regex_flags) assert all(self.callback.values()) for type_, f in self.user_callbacks.items(): @@ -331,7 +331,7 @@ class TraditionalLexer(Lexer): else: self.callback[type_] = f - self.mres = build_mres(terminals, global_flags) + self.mres = build_mres(terminals, g_regex_flags) def match(self, stream, pos): for mre, type_from_index in self.mres: @@ -347,7 +347,7 @@ class TraditionalLexer(Lexer): class ContextualLexer(Lexer): - def __init__(self, terminals, states, ignore=(), always_accept=(), user_callbacks={}, global_flags=0): + def __init__(self, terminals, states, ignore=(), always_accept=(), user_callbacks={}, g_regex_flags=0): tokens_by_name = {} for t in terminals: assert t.name not in tokens_by_name, t @@ -362,12 +362,12 @@ class ContextualLexer(Lexer): except KeyError: accepts = set(accepts) | set(ignore) | set(always_accept) state_tokens = [tokens_by_name[n] for n in accepts if n and n in tokens_by_name] - lexer = TraditionalLexer(state_tokens, ignore=ignore, user_callbacks=user_callbacks, global_flags=global_flags) + lexer = TraditionalLexer(state_tokens, ignore=ignore, user_callbacks=user_callbacks, g_regex_flags=g_regex_flags) lexer_by_tokens[key] = lexer self.lexers[state] = lexer - self.root_lexer = TraditionalLexer(terminals, ignore=ignore, user_callbacks=user_callbacks, global_flags=global_flags) + self.root_lexer = TraditionalLexer(terminals, ignore=ignore, user_callbacks=user_callbacks, g_regex_flags=g_regex_flags) def lex(self, stream, get_parser_state): parser_state = get_parser_state() diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index 6e51489..d68d186 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -88,7 +88,7 @@ class WithLexer(_ParserFrontend): return self._parse(token_stream, start) def init_traditional_lexer(self): - self.lexer = TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks, global_flags=self.lexer_conf.global_flags) + self.lexer = TraditionalLexer(self.lexer_conf.tokens, ignore=self.lexer_conf.ignore, user_callbacks=self.lexer_conf.callbacks, g_regex_flags=self.lexer_conf.g_regex_flags) class LALR_WithLexer(WithLexer): def __init__(self, lexer_conf, parser_conf, options=None): @@ -113,7 +113,7 @@ class LALR_ContextualLexer(LALR_WithLexer): ignore=self.lexer_conf.ignore, always_accept=always_accept, user_callbacks=self.lexer_conf.callbacks, - global_flags=self.lexer_conf.global_flags) + g_regex_flags=self.lexer_conf.g_regex_flags) def parse(self, text, start=None): @@ -188,7 +188,7 @@ class XEarley(_ParserFrontend): if width == 0: raise ValueError("Dynamic Earley doesn't allow zero-width regexps", t) - self.regexps[t.name] = re.compile(regexp, lexer_conf.global_flags) + self.regexps[t.name] = re.compile(regexp, lexer_conf.g_regex_flags) def parse(self, text, start): return self._parse(text, start) diff --git a/lark_stubs/lark.pyi b/lark_stubs/lark.pyi index ae297d0..76a6a54 100644 --- a/lark_stubs/lark.pyi +++ b/lark_stubs/lark.pyi @@ -34,7 +34,7 @@ class LarkOptions: maybe_placeholders: bool lexer_callbacks: Dict[str, Callable[[Token], Token]] cache_grammar: bool - global_flags: int + g_regex_flags: int class Lark: @@ -58,7 +58,7 @@ class Lark: propagate_positions: bool = False, maybe_placeholders: bool = False, lexer_callbacks: Optional[Dict[str, Callable[[Token], Token]]] = None, - global_flags: int = ... + g_regex_flags: int = ... ): ... diff --git a/lark_stubs/lexer.pyi b/lark_stubs/lexer.pyi index f72edd2..a43b754 100644 --- a/lark_stubs/lexer.pyi +++ b/lark_stubs/lexer.pyi @@ -113,7 +113,7 @@ class TraditionalLexer(Lexer): terminals: Collection[TerminalDef], ignore: Collection[str] = ..., user_callbacks: Dict[str, _Callback] = ..., - global_flags: int = ... + g_regex_flags: int = ... ): ... @@ -138,7 +138,7 @@ class ContextualLexer(Lexer): ignore: Collection[str] = ..., always_accept: Collection[str] = ..., user_callbacks: Dict[str, _Callback] = ..., - global_flags: int = ... + g_regex_flags: int = ... ): ... diff --git a/tests/test_parser.py b/tests/test_parser.py index 80fbbfd..a2275ad 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -539,7 +539,7 @@ class CustomLexer(Lexer): so it uses the traditionalparser as implementation without custom lexing behaviour. """ def __init__(self, lexer_conf): - self.lexer = TraditionalLexer(lexer_conf.tokens, ignore=lexer_conf.ignore, user_callbacks=lexer_conf.callbacks, global_flags=lexer_conf.global_flags) + self.lexer = TraditionalLexer(lexer_conf.tokens, ignore=lexer_conf.ignore, user_callbacks=lexer_conf.callbacks, g_regex_flags=lexer_conf.g_regex_flags) def lex(self, *args, **kwargs): return self.lexer.lex(*args, **kwargs) @@ -833,13 +833,13 @@ def _make_parser_test(LEXER, PARSER): x = g.parse("starts") self.assertSequenceEqual(x.children, ['starts']) - def test_global_flags(self): + def test_g_regex_flags(self): g = _Lark(""" start: "a" /b+/ C C: "C" | D D: "D" E E: "e" - """, global_flags=re.I) + """, g_regex_flags=re.I) x1 = g.parse("ABBc") x2 = g.parse("abdE")