Browse Source

Minor fixes in lexer

tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.5.3
Erez Shinan 6 years ago
parent
commit
8acd77d7ff
2 changed files with 3 additions and 2 deletions
  1. +1
    -0
      lark/grammars/common.g
  2. +2
    -2
      lark/lexer.py

+ 1
- 0
lark/grammars/common.g View File

@@ -12,6 +12,7 @@ DECIMAL: INT "." INT? | "." INT
// float = /-?\d+(\.\d+)?([eE][+-]?\d+)?/
_EXP: ("e"|"E") SIGNED_INT
FLOAT: INT _EXP | DECIMAL _EXP?
SIGNED_FLOAT: ["+"|"-"] INT

NUMBER: FLOAT | INT
SIGNED_NUMBER: ["+"|"-"] NUMBER


+ 2
- 2
lark/lexer.py View File

@@ -79,7 +79,7 @@ class _Lex:

def lex(self, stream, newline_types, ignore_types):
newline_types = list(newline_types)
newline_types = list(newline_types)
ignore_types = list(ignore_types)
line_ctr = LineCounter()

while True:
@@ -93,7 +93,7 @@ class _Lex:
t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
if t.type in lexer.callback:
t = lexer.callback[t.type](t)
lexer = yield t
yield t

line_ctr.feed(value, type_ in newline_types)
break


Loading…
Cancel
Save