@@ -5,7 +5,7 @@ | |||||
from lark import Lark, InlineTransformer | from lark import Lark, InlineTransformer | ||||
try: | try: | ||||
input = raw_input | |||||
input = raw_input # For Python2 compatibility | |||||
except NameError: | except NameError: | ||||
pass | pass | ||||
@@ -1,14 +1,15 @@ | |||||
# | # | ||||
# This example demonstrates scanless parsing using the earley_nolex frontend | |||||
# This example demonstrates scanless parsing using the dynamic-lexer earley frontend | |||||
# | # | ||||
# Using a lexer for configuration files is tricky, because values don't | # Using a lexer for configuration files is tricky, because values don't | ||||
# have to be surrounded by delimiters. | |||||
# In this example with skip lexing and let the Earley parser resolve the ambiguity. | |||||
# have to be surrounded by delimiters. Using a standard lexer for this just won't work. | |||||
# | |||||
# In this example we use a dynamic lexer and let the Earley parser resolve the ambiguity. | |||||
# | # | ||||
# Future versions of lark will make it easier to write these kinds of grammars. | # Future versions of lark will make it easier to write these kinds of grammars. | ||||
# | # | ||||
# Another approach is to use the contextual lexer. It is less powerful than the scanless approach, | |||||
# but it can handle some ambiguity in lexing and it's much faster since it uses LALR(1). | |||||
# Another approach is to use the contextual lexer with LALR. It is less powerful than Earley, | |||||
# but it can handle some ambiguity when lexing and it's much faster. | |||||
# See examples/conf.py for an example of that approach. | # See examples/conf.py for an example of that approach. | ||||
# | # | ||||
@@ -25,7 +26,7 @@ parser = Lark(r""" | |||||
%import common.WS_INLINE | %import common.WS_INLINE | ||||
%ignore WS_INLINE | %ignore WS_INLINE | ||||
""", lexer=None) | |||||
""", lexer='dynamic') | |||||
def test(): | def test(): | ||||
sample_conf = """ | sample_conf = """ | ||||
@@ -25,7 +25,7 @@ test_json = ''' | |||||
def test_scanless(): | def test_scanless(): | ||||
json_parser = Lark(json_grammar) | |||||
json_parser = Lark(json_grammar, lexer=None) | |||||
tree = json_parser.parse(test_json) | tree = json_parser.parse(test_json) | ||||
# print ('@@', tree.pretty()) | # print ('@@', tree.pretty()) | ||||
@@ -1,5 +1,10 @@ | |||||
# This example implements a LOGO-like toy language for Python's turtle, with interpreter. | # This example implements a LOGO-like toy language for Python's turtle, with interpreter. | ||||
try: | |||||
input = raw_input # For Python2 compatibility | |||||
except NameError: | |||||
pass | |||||
import turtle | import turtle | ||||
from lark import Lark | from lark import Lark | ||||
@@ -76,5 +81,5 @@ def test(): | |||||
run_turtle(text) | run_turtle(text) | ||||
if __name__ == '__main__': | if __name__ == '__main__': | ||||
#test | |||||
# test() | |||||
main() | main() |
@@ -3,4 +3,4 @@ from .common import ParseError, GrammarError | |||||
from .lark import Lark | from .lark import Lark | ||||
from .utils import inline_args | from .utils import inline_args | ||||
__version__ = "0.2.10" | |||||
__version__ = "0.3.0" |
@@ -129,7 +129,7 @@ class Lark: | |||||
if self.options.parser == 'lalr': | if self.options.parser == 'lalr': | ||||
self.options.lexer = 'standard' | self.options.lexer = 'standard' | ||||
elif self.options.parser == 'earley': | elif self.options.parser == 'earley': | ||||
self.options.lexer = None | |||||
self.options.lexer = 'dynamic' | |||||
else: | else: | ||||
assert False, self.options.parser | assert False, self.options.parser | ||||
lexer = self.options.lexer | lexer = self.options.lexer | ||||
@@ -24,7 +24,7 @@ from ..common import ParseError, UnexpectedToken, Terminal | |||||
from ..tree import Tree | from ..tree import Tree | ||||
from .grammar_analysis import GrammarAnalyzer | from .grammar_analysis import GrammarAnalyzer | ||||
from earley import ResolveAmbig, ApplyCallbacks, Item, NewsList, Derivation, END_TOKEN, Column | |||||
from .earley import ResolveAmbig, ApplyCallbacks, Item, NewsList, Derivation, END_TOKEN, Column | |||||
class Parser: | class Parser: | ||||
def __init__(self, rules, start_symbol, callback, resolve_ambiguity=True, ignore=()): | def __init__(self, rules, start_symbol, callback, resolve_ambiguity=True, ignore=()): | ||||
@@ -86,7 +86,7 @@ class Reconstructor: | |||||
MatchTerminal(sym) if is_terminal(sym) else MatchTree(sym) | MatchTerminal(sym) if is_terminal(sym) else MatchTree(sym) | ||||
for sym in expansion if not is_discarded_terminal(sym)] | for sym in expansion if not is_discarded_terminal(sym)] | ||||
rules.append((name, reduced, WriteTokens(name, expansion).f)) | |||||
rules.append((name, reduced, WriteTokens(name, expansion).f, None)) | |||||
self.rules = rules | self.rules = rules | ||||