From 8acd77d7ffb546e2aa55c80042a8908d7e7e2fc9 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Tue, 16 Jan 2018 00:51:30 +0200 Subject: [PATCH] Minor fixes in lexer --- lark/grammars/common.g | 1 + lark/lexer.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lark/grammars/common.g b/lark/grammars/common.g index a54d49d..c38f485 100644 --- a/lark/grammars/common.g +++ b/lark/grammars/common.g @@ -12,6 +12,7 @@ DECIMAL: INT "." INT? | "." INT // float = /-?\d+(\.\d+)?([eE][+-]?\d+)?/ _EXP: ("e"|"E") SIGNED_INT FLOAT: INT _EXP | DECIMAL _EXP? +SIGNED_FLOAT: ["+"|"-"] INT NUMBER: FLOAT | INT SIGNED_NUMBER: ["+"|"-"] NUMBER diff --git a/lark/lexer.py b/lark/lexer.py index 844025d..64cfb46 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -79,7 +79,7 @@ class _Lex: def lex(self, stream, newline_types, ignore_types): newline_types = list(newline_types) - newline_types = list(newline_types) + ignore_types = list(ignore_types) line_ctr = LineCounter() while True: @@ -93,7 +93,7 @@ class _Lex: t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) if t.type in lexer.callback: t = lexer.callback[t.type](t) - lexer = yield t + yield t line_ctr.feed(value, type_ in newline_types) break