diff --git a/lark/__init__.py b/lark/__init__.py index 6ca88da..f056182 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -7,4 +7,4 @@ from .exceptions import (ParseError, LexError, GrammarError, UnexpectedToken, from .lexer import Token from .lark import Lark -__version__ = "0.11.3" +__version__ = "0.11.4" diff --git a/lark/lark.py b/lark/lark.py index 6c6a239..b50edc5 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -573,7 +573,7 @@ class Lark(Serialize): @property def source(self): - warn("Lark.source attribute has been renamed to Lark.source_path", DeprecationWarning) + warn("Attribute Lark.source was renamed to Lark.source_path", DeprecationWarning) return self.source_path @source.setter @@ -582,7 +582,7 @@ class Lark(Serialize): @property def grammar_source(self): - warn("Lark.grammar_source attribute has been renamed to Lark.source_grammar", DeprecationWarning) + warn("Attribute Lark.grammar_source was renamed to Lark.source_grammar", DeprecationWarning) return self.source_grammar @grammar_source.setter diff --git a/lark/lexer.py b/lark/lexer.py index 4d18704..a2aefd2 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -6,6 +6,7 @@ from .utils import Str, classify, get_regexp_width, Py36, Serialize, suppress from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken ###{standalone +from warnings import warn from copy import copy @@ -128,9 +129,9 @@ class Token(Str): end_column will be 5. end_pos: the index where the token ends (basically ``pos_in_stream + len(token)``) """ - __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') + __slots__ = ('type', 'start_pos', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') - def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): + def __new__(cls, type_, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None, pos_in_stream=None): try: self = super(Token, cls).__new__(cls, value) except UnicodeDecodeError: @@ -138,7 +139,7 @@ class Token(Str): self = super(Token, cls).__new__(cls, value) self.type = type_ - self.pos_in_stream = pos_in_stream + self.start_pos = start_pos if start_pos is not None else pos_in_stream self.value = value self.line = line self.column = column @@ -147,6 +148,11 @@ class Token(Str): self.end_pos = end_pos return self + @property + def pos_in_stream(self): + warn("Attribute Token.pos_in_stream was renamed to Token.start_pos", DeprecationWarning) + return self.start_pos + def update(self, type_=None, value=None): return Token.new_borrow_pos( type_ if type_ is not None else self.type, @@ -156,16 +162,16 @@ class Token(Str): @classmethod def new_borrow_pos(cls, type_, value, borrow_t): - return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) + return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) def __reduce__(self): - return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column)) + return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column)) def __repr__(self): return 'Token(%r, %r)' % (self.type, self.value) def __deepcopy__(self, memo): - return Token(self.type, self.value, self.pos_in_stream, self.line, self.column) + return Token(self.type, self.value, self.start_pos, self.line, self.column) def __eq__(self, other): if isinstance(other, Token) and self.type != other.type: diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 569761a..e7a2af3 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -21,6 +21,15 @@ class ExpandSingleChild: return self.node_builder(children) + +def _pp_get_meta(children): + for c in children: + if isinstance(c, Tree): + if not c.meta.empty: + return c.meta + elif isinstance(c, Token): + return c + class PropagatePositions: def __init__(self, node_builder): self.node_builder = node_builder @@ -31,37 +40,20 @@ class PropagatePositions: # local reference to Tree.meta reduces number of presence checks if isinstance(res, Tree): res_meta = res.meta - for c in children: - if isinstance(c, Tree): - child_meta = c.meta - if not child_meta.empty: - res_meta.line = child_meta.line - res_meta.column = child_meta.column - res_meta.start_pos = child_meta.start_pos - res_meta.empty = False - break - elif isinstance(c, Token): - res_meta.line = c.line - res_meta.column = c.column - res_meta.start_pos = c.pos_in_stream - res_meta.empty = False - break - - for c in reversed(children): - if isinstance(c, Tree): - child_meta = c.meta - if not child_meta.empty: - res_meta.end_line = child_meta.end_line - res_meta.end_column = child_meta.end_column - res_meta.end_pos = child_meta.end_pos - res_meta.empty = False - break - elif isinstance(c, Token): - res_meta.end_line = c.end_line - res_meta.end_column = c.end_column - res_meta.end_pos = c.end_pos - res_meta.empty = False - break + + src_meta = _pp_get_meta(children) + if src_meta: + res_meta.line = src_meta.line + res_meta.column = src_meta.column + res_meta.start_pos = src_meta.start_pos + res_meta.empty = False + + src_meta = _pp_get_meta(reversed(children)) + if src_meta: + res_meta.end_line = src_meta.end_line + res_meta.end_column = src_meta.end_column + res_meta.end_pos = src_meta.end_pos + res_meta.empty = False return res