Browse Source

Added the experimental "propagate_positions" feature (only for standard lexer for now).

tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.5.1
Erez Shinan 7 years ago
parent
commit
a588a70a7a
4 changed files with 62 additions and 5 deletions
  1. +3
    -1
      lark/lark.py
  2. +13
    -3
      lark/lexer.py
  3. +26
    -1
      lark/parse_tree_builder.py
  4. +20
    -0
      lark/utils.py

+ 3
- 1
lark/lark.py View File

@@ -39,6 +39,7 @@ class LarkOptions(object):
postlex - Lexer post-processing (Default: None)
start - The start symbol (Default: start)
profile - Measure run-time usage in Lark. Read results from the profiler proprety (Default: False)
propagate_positions - Experimental. Don't use yet.
"""
__doc__ += OPTIONS_DOC
def __init__(self, options_dict):
@@ -55,6 +56,7 @@ class LarkOptions(object):
self.start = o.pop('start', 'start')
self.profile = o.pop('profile', False)
self.ambiguity = o.pop('ambiguity', 'auto')
self.propagate_positions = o.pop('propagate_positions', False)

assert self.parser in ('earley', 'lalr', None)

@@ -160,7 +162,7 @@ class Lark:

def _build_parser(self):
self.parser_class = get_frontend(self.options.parser, self.options.lexer)
self.parse_tree_builder = ParseTreeBuilder(self.options.tree_class)
self.parse_tree_builder = ParseTreeBuilder(self.options.tree_class, self.options.propagate_positions)
rules, callback = self.parse_tree_builder.create_tree_builder(self.rules, self.options.transformer)
if self.profiler:
for f in dir(callback):


+ 13
- 3
lark/lexer.py View File

@@ -155,17 +155,27 @@ class Lexer(object):
if m:
value = m.group(0)
type_ = type_from_index[m.lastindex]
if type_ not in ignore_types:
to_yield = type_ not in ignore_types

if to_yield:
t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
if t.type in self.callback:
t = self.callback[t.type](t)
yield t

end_col = t.column + len(value)
if type_ in newline_types:
newlines = value.count(self.newline_char)
if newlines:
line += newlines
col_start_pos = lex_pos + value.rindex(self.newline_char)
last_newline_index = value.rindex(self.newline_char) + 1
col_start_pos = lex_pos + last_newline_index
end_col = len(value) - last_newline_index

if to_yield:
t.end_line = line
t.end_col = end_col
yield t

lex_pos += len(value)
break
else:


+ 26
- 1
lark/parse_tree_builder.py View File

@@ -1,4 +1,5 @@
from .common import is_terminal, GrammarError
from .utils import suppress
from .lexer import Token

class Callback(object):
@@ -42,10 +43,31 @@ def create_rule_handler(expansion, usermethod, keep_all_tokens, filter_out):
# else, if no filtering required..
return usermethod

def propagate_positions_wrapper(f):
def _f(args):
res = f(args)

if args:
for a in args:
with suppress(AttributeError):
res.line = a.line
res.column = a.column
break

for a in reversed(args):
with suppress(AttributeError):
res.end_line = a.end_line
res.end_col = a.end_col
break

return res

return _f

class ParseTreeBuilder:
def __init__(self, tree_class):
def __init__(self, tree_class, propagate_positions=False):
self.tree_class = tree_class
self.propagate_positions = propagate_positions

def _create_tree_builder_function(self, name):
tree_class = self.tree_class
@@ -92,6 +114,9 @@ class ParseTreeBuilder:

alias_handler = create_rule_handler(expansion, f, keep_all_tokens, filter_out)

if self.propagate_positions:
alias_handler = propagate_positions_wrapper(alias_handler)

callback_name = 'autoalias_%s_%s' % (_origin, '_'.join(expansion))
if hasattr(callback, callback_name):
raise GrammarError("Rule expansion '%s' already exists in rule %s" % (' '.join(expansion), origin))


+ 20
- 0
lark/utils.py View File

@@ -1,6 +1,7 @@
import functools
import types
from collections import deque
from contextlib import contextmanager

class fzset(frozenset):
def __repr__(self):
@@ -88,5 +89,24 @@ except NameError:
return -1


try:
from contextlib import suppress # Python 3
except ImportError:
@contextmanager
def suppress(*excs):
'''Catch and dismiss the provided exception

>>> x = 'hello'
>>> with suppress(IndexError):
... x = x[10]
>>> x
'hello'
'''
try:
yield
except excs:
pass





Loading…
Cancel
Save