@@ -39,6 +39,7 @@ class LarkOptions(object): | |||||
postlex - Lexer post-processing (Default: None) | postlex - Lexer post-processing (Default: None) | ||||
start - The start symbol (Default: start) | start - The start symbol (Default: start) | ||||
profile - Measure run-time usage in Lark. Read results from the profiler proprety (Default: False) | profile - Measure run-time usage in Lark. Read results from the profiler proprety (Default: False) | ||||
propagate_positions - Experimental. Don't use yet. | |||||
""" | """ | ||||
__doc__ += OPTIONS_DOC | __doc__ += OPTIONS_DOC | ||||
def __init__(self, options_dict): | def __init__(self, options_dict): | ||||
@@ -55,6 +56,7 @@ class LarkOptions(object): | |||||
self.start = o.pop('start', 'start') | self.start = o.pop('start', 'start') | ||||
self.profile = o.pop('profile', False) | self.profile = o.pop('profile', False) | ||||
self.ambiguity = o.pop('ambiguity', 'auto') | self.ambiguity = o.pop('ambiguity', 'auto') | ||||
self.propagate_positions = o.pop('propagate_positions', False) | |||||
assert self.parser in ('earley', 'lalr', None) | assert self.parser in ('earley', 'lalr', None) | ||||
@@ -160,7 +162,7 @@ class Lark: | |||||
def _build_parser(self): | def _build_parser(self): | ||||
self.parser_class = get_frontend(self.options.parser, self.options.lexer) | self.parser_class = get_frontend(self.options.parser, self.options.lexer) | ||||
self.parse_tree_builder = ParseTreeBuilder(self.options.tree_class) | |||||
self.parse_tree_builder = ParseTreeBuilder(self.options.tree_class, self.options.propagate_positions) | |||||
rules, callback = self.parse_tree_builder.create_tree_builder(self.rules, self.options.transformer) | rules, callback = self.parse_tree_builder.create_tree_builder(self.rules, self.options.transformer) | ||||
if self.profiler: | if self.profiler: | ||||
for f in dir(callback): | for f in dir(callback): | ||||
@@ -155,17 +155,27 @@ class Lexer(object): | |||||
if m: | if m: | ||||
value = m.group(0) | value = m.group(0) | ||||
type_ = type_from_index[m.lastindex] | type_ = type_from_index[m.lastindex] | ||||
if type_ not in ignore_types: | |||||
to_yield = type_ not in ignore_types | |||||
if to_yield: | |||||
t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos) | t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos) | ||||
if t.type in self.callback: | if t.type in self.callback: | ||||
t = self.callback[t.type](t) | t = self.callback[t.type](t) | ||||
yield t | |||||
end_col = t.column + len(value) | |||||
if type_ in newline_types: | if type_ in newline_types: | ||||
newlines = value.count(self.newline_char) | newlines = value.count(self.newline_char) | ||||
if newlines: | if newlines: | ||||
line += newlines | line += newlines | ||||
col_start_pos = lex_pos + value.rindex(self.newline_char) | |||||
last_newline_index = value.rindex(self.newline_char) + 1 | |||||
col_start_pos = lex_pos + last_newline_index | |||||
end_col = len(value) - last_newline_index | |||||
if to_yield: | |||||
t.end_line = line | |||||
t.end_col = end_col | |||||
yield t | |||||
lex_pos += len(value) | lex_pos += len(value) | ||||
break | break | ||||
else: | else: | ||||
@@ -1,4 +1,5 @@ | |||||
from .common import is_terminal, GrammarError | from .common import is_terminal, GrammarError | ||||
from .utils import suppress | |||||
from .lexer import Token | from .lexer import Token | ||||
class Callback(object): | class Callback(object): | ||||
@@ -42,10 +43,31 @@ def create_rule_handler(expansion, usermethod, keep_all_tokens, filter_out): | |||||
# else, if no filtering required.. | # else, if no filtering required.. | ||||
return usermethod | return usermethod | ||||
def propagate_positions_wrapper(f): | |||||
def _f(args): | |||||
res = f(args) | |||||
if args: | |||||
for a in args: | |||||
with suppress(AttributeError): | |||||
res.line = a.line | |||||
res.column = a.column | |||||
break | |||||
for a in reversed(args): | |||||
with suppress(AttributeError): | |||||
res.end_line = a.end_line | |||||
res.end_col = a.end_col | |||||
break | |||||
return res | |||||
return _f | |||||
class ParseTreeBuilder: | class ParseTreeBuilder: | ||||
def __init__(self, tree_class): | |||||
def __init__(self, tree_class, propagate_positions=False): | |||||
self.tree_class = tree_class | self.tree_class = tree_class | ||||
self.propagate_positions = propagate_positions | |||||
def _create_tree_builder_function(self, name): | def _create_tree_builder_function(self, name): | ||||
tree_class = self.tree_class | tree_class = self.tree_class | ||||
@@ -92,6 +114,9 @@ class ParseTreeBuilder: | |||||
alias_handler = create_rule_handler(expansion, f, keep_all_tokens, filter_out) | alias_handler = create_rule_handler(expansion, f, keep_all_tokens, filter_out) | ||||
if self.propagate_positions: | |||||
alias_handler = propagate_positions_wrapper(alias_handler) | |||||
callback_name = 'autoalias_%s_%s' % (_origin, '_'.join(expansion)) | callback_name = 'autoalias_%s_%s' % (_origin, '_'.join(expansion)) | ||||
if hasattr(callback, callback_name): | if hasattr(callback, callback_name): | ||||
raise GrammarError("Rule expansion '%s' already exists in rule %s" % (' '.join(expansion), origin)) | raise GrammarError("Rule expansion '%s' already exists in rule %s" % (' '.join(expansion), origin)) | ||||
@@ -1,6 +1,7 @@ | |||||
import functools | import functools | ||||
import types | import types | ||||
from collections import deque | from collections import deque | ||||
from contextlib import contextmanager | |||||
class fzset(frozenset): | class fzset(frozenset): | ||||
def __repr__(self): | def __repr__(self): | ||||
@@ -88,5 +89,24 @@ except NameError: | |||||
return -1 | return -1 | ||||
try: | |||||
from contextlib import suppress # Python 3 | |||||
except ImportError: | |||||
@contextmanager | |||||
def suppress(*excs): | |||||
'''Catch and dismiss the provided exception | |||||
>>> x = 'hello' | |||||
>>> with suppress(IndexError): | |||||
... x = x[10] | |||||
>>> x | |||||
'hello' | |||||
''' | |||||
try: | |||||
yield | |||||
except excs: | |||||
pass | |||||