diff --git a/examples/advanced/python_bytecode.py b/examples/advanced/python_bytecode.py deleted file mode 100644 index 6165e82..0000000 --- a/examples/advanced/python_bytecode.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -Compile Python to Bytecode -========================== - -A toy example that compiles Python directly to bytecode, without generating an AST. -It currently only works for very very simple Python code. - -It requires the 'bytecode' library. You can get it using -:: - - $ pip install bytecode - -""" -from lark import Lark, Transformer, v_args -from lark.indenter import Indenter - -from bytecode import Instr, Bytecode - -class PythonIndenter(Indenter): - NL_type = '_NEWLINE' - OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE'] - CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE'] - INDENT_type = '_INDENT' - DEDENT_type = '_DEDENT' - tab_len = 8 - - -@v_args(inline=True) -class Compile(Transformer): - def number(self, n): - return [Instr('LOAD_CONST', int(n))] - def string(self, s): - return [Instr('LOAD_CONST', s[1:-1])] - def var(self, n): - return [Instr('LOAD_NAME', n)] - - def arith_expr(self, a, op, b): - # TODO support chain arithmetic - assert op == '+' - return a + b + [Instr('BINARY_ADD')] - - def arguments(self, args): - return args - - def funccall(self, name, args): - return name + args + [Instr('CALL_FUNCTION', 1)] - - @v_args(inline=False) - def file_input(self, stmts): - return sum(stmts, []) + [Instr("RETURN_VALUE")] - - def expr_stmt(self, lval, rval): - # TODO more complicated than that - name ,= lval - assert name.name == 'LOAD_NAME' # XXX avoid with another layer of abstraction - return rval + [Instr("STORE_NAME", name.arg)] - - def __default__(self, *args): - assert False, args - - -python_parser3 = Lark.open('python3.lark', rel_to=__file__, start='file_input', - parser='lalr', postlex=PythonIndenter(), - transformer=Compile(), propagate_positions=False) - -def compile_python(s): - insts = python_parser3.parse(s+"\n") - return Bytecode(insts).to_code() - -code = compile_python(""" -a = 3 -b = 5 -print("Hello World!") -print(a+(b+2)) -print((a+b)+2) -""") -exec(code) -# -- Output -- -# Hello World! -# 10 -# 10 diff --git a/lark/lark.py b/lark/lark.py index 45dec4d..744cf4b 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -340,7 +340,9 @@ class Lark(Serialize): if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS: raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS)) - if self.options.postlex is not None: + if self.options.parser is None: + terminals_to_keep = '*' + elif self.options.postlex is not None: terminals_to_keep = set(self.options.postlex.always_accept) else: terminals_to_keep = set() diff --git a/lark/load_grammar.py b/lark/load_grammar.py index abcfce1..309826b 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -782,12 +782,13 @@ class Grammar: break # Filter out unused terminals - used_terms = {t.name for r in compiled_rules - for t in r.expansion - if isinstance(t, Terminal)} - terminals, unused = classify_bool(terminals, lambda t: t.name in used_terms or t.name in self.ignore or t.name in terminals_to_keep) - if unused: - logger.debug("Unused terminals: %s", [t.name for t in unused]) + if terminals_to_keep != '*': + used_terms = {t.name for r in compiled_rules + for t in r.expansion + if isinstance(t, Terminal)} + terminals, unused = classify_bool(terminals, lambda t: t.name in used_terms or t.name in self.ignore or t.name in terminals_to_keep) + if unused: + logger.debug("Unused terminals: %s", [t.name for t in unused]) return terminals, compiled_rules, self.ignore