diff --git a/README.md b/README.md index 76dcdea..156a671 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ Check out the [JSON tutorial](/docs/json_tutorial.md#conclusion) for more detail ### Projects using Lark + - [Poetry](https://github.com/python-poetry/poetry-core) - A utility for dependency management and packaging - [tartiflette](https://github.com/dailymotion/tartiflette) - a GraphQL server by Dailymotion - [Hypothesis](https://github.com/HypothesisWorks/hypothesis) - Library for property-based testing - [mappyfile](https://github.com/geographika/mappyfile) - a MapFile parser for working with MapServer configuration diff --git a/lark-stubs/tree.pyi b/lark-stubs/tree.pyi index ea99ff6..0c12819 100644 --- a/lark-stubs/tree.pyi +++ b/lark-stubs/tree.pyi @@ -40,6 +40,9 @@ class Tree: def expand_kids_by_index(self, *indices: int) -> None: ... + def expand_kids_by_data(self, *data_values: str) -> bool: + ... + def scan_values(self, pred: Callable[[Union[str, Tree]], bool]) -> Iterator[str]: ... diff --git a/lark/exceptions.py b/lark/exceptions.py index fdcd52b..9f18753 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -210,7 +210,7 @@ class UnexpectedToken(ParseError, UnexpectedInput): # TODO considered_rules and expected can be figured out using state self.line = getattr(token, 'line', '?') self.column = getattr(token, 'column', '?') - self.pos_in_stream = getattr(token, 'pos_in_stream', None) + self.pos_in_stream = getattr(token, 'start_pos', None) self.state = state self.token = token diff --git a/lark/lexer.py b/lark/lexer.py index 7c2f979..a82cc18 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -150,7 +150,7 @@ class Token(Str): @property def pos_in_stream(self): - warn("Attribute Token.pos_in_stream was renamed to Token.start_pos", DeprecationWarning) + warn("Attribute Token.pos_in_stream was renamed to Token.start_pos", DeprecationWarning, 2) return self.start_pos def update(self, type_=None, value=None): diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 69bd788..7c64196 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -91,6 +91,7 @@ TERMINALS = { 'STRING': r'"(\\"|\\\\|[^"\n])*?"i?', 'REGEXP': r'/(?!/)(\\/|\\\\|[^/])*?/[%s]*' % _RE_FLAGS, '_NL': r'(\r?\n)+\s*', + '_NL_OR': r'(\r?\n)+\s*\|', 'WS': r'[ \t]+', 'COMMENT': r'\s*//[^\n]*', '_TO': '->', @@ -113,9 +114,10 @@ RULES = { ''], '_template_params': ['RULE', '_template_params _COMMA RULE'], - 'expansions': ['alias', - 'expansions _OR alias', - 'expansions _NL _OR alias'], + 'expansions': ['_expansions'], + '_expansions': ['alias', + '_expansions _OR alias', + '_expansions _NL_OR alias'], '?alias': ['expansion _TO RULE', 'expansion'], 'expansion': ['_expansion'], @@ -356,12 +358,8 @@ class SimplifyRule_Visitor(Visitor): @staticmethod def _flatten(tree): - while True: - to_expand = [i for i, child in enumerate(tree.children) - if isinstance(child, Tree) and child.data == tree.data] - if not to_expand: - break - tree.expand_kids_by_index(*to_expand) + while tree.expand_kids_by_data(tree.data): + pass def expansion(self, tree): # rules_list unpacking @@ -599,8 +597,7 @@ def _make_joined_pattern(regexp, flags_set): return PatternRE(regexp, flags) - -class TerminalTreeToPattern(Transformer): +class TerminalTreeToPattern(Transformer_NonRecursive): def pattern(self, ps): p ,= ps return p @@ -670,8 +667,8 @@ class Grammar: def compile(self, start, terminals_to_keep): # We change the trees in-place (to support huge grammars) # So deepcopy allows calling compile more than once. - term_defs = deepcopy(list(self.term_defs)) - rule_defs = [(n,p,nr_deepcopy_tree(t),o) for n,p,t,o in self.rule_defs] + term_defs = [(n, (nr_deepcopy_tree(t), p)) for n, (t, p) in self.term_defs] + rule_defs = [(n, p, nr_deepcopy_tree(t), o) for n, p, t, o in self.rule_defs] # =================== # Compile Terminals @@ -919,7 +916,7 @@ def _get_parser(): parser_conf = ParserConf(rules, callback, ['start']) lexer_conf.lexer_type = 'standard' parser_conf.parser_type = 'lalr' - _get_parser.cache = ParsingFrontend(lexer_conf, parser_conf, {}) + _get_parser.cache = ParsingFrontend(lexer_conf, parser_conf, None) return _get_parser.cache GRAMMAR_ERRORS = [ @@ -1096,9 +1093,7 @@ class GrammarBuilder: # TODO: think about what to do with 'options' base = self._definitions[name][1] - while len(base.children) == 2: - assert isinstance(base.children[0], Tree) and base.children[0].data == 'expansions', base - base = base.children[0] + assert isinstance(base, Tree) and base.data == 'expansions' base.children.insert(0, exp) def _ignore(self, exp_or_name): diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 286038e..fa526b0 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -204,8 +204,7 @@ class AmbiguousExpander: if i in self.to_expand: ambiguous.append(i) - to_expand = [j for j, grandchild in enumerate(child.children) if _is_ambig_tree(grandchild)] - child.expand_kids_by_index(*to_expand) + child.expand_kids_by_data('_ambig') if not ambiguous: return self.node_builder(children) diff --git a/lark/tree.py b/lark/tree.py index bee53cf..0937b85 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -107,6 +107,17 @@ class Tree(object): kid = self.children[i] self.children[i:i+1] = kid.children + def expand_kids_by_data(self, *data_values): + """Expand (inline) children with any of the given data values. Returns True if anything changed""" + changed = False + for i in range(len(self.children)-1, -1, -1): + child = self.children[i] + if isinstance(child, Tree) and child.data in data_values: + self.children[i:i+1] = child.children + changed = True + return changed + + def scan_values(self, pred): """Return all values in the tree that evaluate pred(value) as true. diff --git a/tests/test_grammar.py b/tests/test_grammar.py index 3ae65f2..47a345c 100644 --- a/tests/test_grammar.py +++ b/tests/test_grammar.py @@ -246,6 +246,18 @@ class TestGrammar(TestCase): self.assertRaises(UnexpectedInput, l.parse, u'A' * 8190) self.assertRaises(UnexpectedInput, l.parse, u'A' * 8192) + def test_large_terminal(self): + # TODO: The `reversed` below is required because otherwise the regex engine is happy + # with just parsing 9 from the string 999 instead of consuming the longest + g = "start: NUMBERS\n" + g += "NUMBERS: " + '|'.join('"%s"' % i for i in reversed(range(0, 1000))) + + l = Lark(g, parser='lalr') + for i in (0, 9, 99, 999): + self.assertEqual(l.parse(str(i)), Tree('start', [str(i)])) + for i in (-1, 1000): + self.assertRaises(UnexpectedInput, l.parse, str(i)) + if __name__ == '__main__': main()