Increased minor version, because it might invalidate serialized objectstags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.12.0
@@ -7,4 +7,4 @@ from .exceptions import (ParseError, LexError, GrammarError, UnexpectedToken, | |||
from .lexer import Token | |||
from .lark import Lark | |||
__version__ = "0.11.3" | |||
__version__ = "0.11.4" |
@@ -573,7 +573,7 @@ class Lark(Serialize): | |||
@property | |||
def source(self): | |||
warn("Lark.source attribute has been renamed to Lark.source_path", DeprecationWarning) | |||
warn("Attribute Lark.source was renamed to Lark.source_path", DeprecationWarning) | |||
return self.source_path | |||
@source.setter | |||
@@ -582,7 +582,7 @@ class Lark(Serialize): | |||
@property | |||
def grammar_source(self): | |||
warn("Lark.grammar_source attribute has been renamed to Lark.source_grammar", DeprecationWarning) | |||
warn("Attribute Lark.grammar_source was renamed to Lark.source_grammar", DeprecationWarning) | |||
return self.source_grammar | |||
@grammar_source.setter | |||
@@ -6,6 +6,7 @@ from .utils import Str, classify, get_regexp_width, Py36, Serialize, suppress | |||
from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken | |||
###{standalone | |||
from warnings import warn | |||
from copy import copy | |||
@@ -128,9 +129,9 @@ class Token(Str): | |||
end_column will be 5. | |||
end_pos: the index where the token ends (basically ``pos_in_stream + len(token)``) | |||
""" | |||
__slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') | |||
__slots__ = ('type', 'start_pos', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') | |||
def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): | |||
def __new__(cls, type_, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None, pos_in_stream=None): | |||
try: | |||
self = super(Token, cls).__new__(cls, value) | |||
except UnicodeDecodeError: | |||
@@ -138,7 +139,7 @@ class Token(Str): | |||
self = super(Token, cls).__new__(cls, value) | |||
self.type = type_ | |||
self.pos_in_stream = pos_in_stream | |||
self.start_pos = start_pos if start_pos is not None else pos_in_stream | |||
self.value = value | |||
self.line = line | |||
self.column = column | |||
@@ -147,6 +148,11 @@ class Token(Str): | |||
self.end_pos = end_pos | |||
return self | |||
@property | |||
def pos_in_stream(self): | |||
warn("Attribute Token.pos_in_stream was renamed to Token.start_pos", DeprecationWarning) | |||
return self.start_pos | |||
def update(self, type_=None, value=None): | |||
return Token.new_borrow_pos( | |||
type_ if type_ is not None else self.type, | |||
@@ -156,16 +162,16 @@ class Token(Str): | |||
@classmethod | |||
def new_borrow_pos(cls, type_, value, borrow_t): | |||
return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) | |||
return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) | |||
def __reduce__(self): | |||
return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column)) | |||
return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column)) | |||
def __repr__(self): | |||
return 'Token(%r, %r)' % (self.type, self.value) | |||
def __deepcopy__(self, memo): | |||
return Token(self.type, self.value, self.pos_in_stream, self.line, self.column) | |||
return Token(self.type, self.value, self.start_pos, self.line, self.column) | |||
def __eq__(self, other): | |||
if isinstance(other, Token) and self.type != other.type: | |||
@@ -21,6 +21,15 @@ class ExpandSingleChild: | |||
return self.node_builder(children) | |||
def _pp_get_meta(children): | |||
for c in children: | |||
if isinstance(c, Tree): | |||
if not c.meta.empty: | |||
return c.meta | |||
elif isinstance(c, Token): | |||
return c | |||
class PropagatePositions: | |||
def __init__(self, node_builder): | |||
self.node_builder = node_builder | |||
@@ -31,37 +40,20 @@ class PropagatePositions: | |||
# local reference to Tree.meta reduces number of presence checks | |||
if isinstance(res, Tree): | |||
res_meta = res.meta | |||
for c in children: | |||
if isinstance(c, Tree): | |||
child_meta = c.meta | |||
if not child_meta.empty: | |||
res_meta.line = child_meta.line | |||
res_meta.column = child_meta.column | |||
res_meta.start_pos = child_meta.start_pos | |||
res_meta.empty = False | |||
break | |||
elif isinstance(c, Token): | |||
res_meta.line = c.line | |||
res_meta.column = c.column | |||
res_meta.start_pos = c.pos_in_stream | |||
res_meta.empty = False | |||
break | |||
for c in reversed(children): | |||
if isinstance(c, Tree): | |||
child_meta = c.meta | |||
if not child_meta.empty: | |||
res_meta.end_line = child_meta.end_line | |||
res_meta.end_column = child_meta.end_column | |||
res_meta.end_pos = child_meta.end_pos | |||
res_meta.empty = False | |||
break | |||
elif isinstance(c, Token): | |||
res_meta.end_line = c.end_line | |||
res_meta.end_column = c.end_column | |||
res_meta.end_pos = c.end_pos | |||
res_meta.empty = False | |||
break | |||
src_meta = _pp_get_meta(children) | |||
if src_meta: | |||
res_meta.line = src_meta.line | |||
res_meta.column = src_meta.column | |||
res_meta.start_pos = src_meta.start_pos | |||
res_meta.empty = False | |||
src_meta = _pp_get_meta(reversed(children)) | |||
if src_meta: | |||
res_meta.end_line = src_meta.end_line | |||
res_meta.end_column = src_meta.end_column | |||
res_meta.end_pos = src_meta.end_pos | |||
res_meta.empty = False | |||
return res | |||