@@ -1,3 +1,5 @@ | |||||
"Parses and creates Grammar objects" | |||||
import os.path | import os.path | ||||
from itertools import chain | from itertools import chain | ||||
import re | import re | ||||
@@ -205,12 +207,6 @@ class SimplifyRule_Visitor(Visitor): | |||||
expansions = _flatten | expansions = _flatten | ||||
def dict_update_safe(d1, d2): | |||||
for k, v in d2.items(): | |||||
assert k not in d1 | |||||
d1[k] = v | |||||
class RuleTreeToText(Transformer): | class RuleTreeToText(Transformer): | ||||
def expansions(self, x): | def expansions(self, x): | ||||
return x | return x | ||||
@@ -233,6 +229,8 @@ class CanonizeTree(InlineTransformer): | |||||
return tokenmods + [value] | return tokenmods + [value] | ||||
class ExtractAnonTokens(InlineTransformer): | class ExtractAnonTokens(InlineTransformer): | ||||
"Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them" | |||||
def __init__(self, tokens): | def __init__(self, tokens): | ||||
self.tokens = tokens | self.tokens = tokens | ||||
self.token_set = {td.name for td in self.tokens} | self.token_set = {td.name for td in self.tokens} | ||||
@@ -244,6 +242,7 @@ class ExtractAnonTokens(InlineTransformer): | |||||
value = p.value | value = p.value | ||||
if p in self.token_reverse and p.flags != self.token_reverse[p].pattern.flags: | if p in self.token_reverse and p.flags != self.token_reverse[p].pattern.flags: | ||||
raise GrammarError(u'Conflicting flags for the same terminal: %s' % p) | raise GrammarError(u'Conflicting flags for the same terminal: %s' % p) | ||||
if isinstance(p, PatternStr): | if isinstance(p, PatternStr): | ||||
try: | try: | ||||
# If already defined, use the user-defined token name | # If already defined, use the user-defined token name | ||||
@@ -353,6 +352,12 @@ def _interleave(l, item): | |||||
def _choice_of_rules(rules): | def _choice_of_rules(rules): | ||||
return T('expansions', [T('expansion', [Token('RULE', name)]) for name in rules]) | return T('expansions', [T('expansion', [Token('RULE', name)]) for name in rules]) | ||||
def dict_update_safe(d1, d2): | |||||
for k, v in d2.items(): | |||||
assert k not in d1 | |||||
d1[k] = v | |||||
class Grammar: | class Grammar: | ||||
def __init__(self, rule_defs, token_defs, ignore): | def __init__(self, rule_defs, token_defs, ignore): | ||||
self.token_defs = token_defs | self.token_defs = token_defs | ||||
@@ -4,10 +4,7 @@ import sre_parse | |||||
from .lexer import Lexer, ContextualLexer, Token | from .lexer import Lexer, ContextualLexer, Token | ||||
from .common import is_terminal, GrammarError, ParserConf, Terminal_Regexp, Terminal_Token | from .common import is_terminal, GrammarError, ParserConf, Terminal_Regexp, Terminal_Token | ||||
from .parsers import lalr_parser, earley | |||||
from .tree import Transformer | |||||
from .parsers import xearley | |||||
from .parsers import lalr_parser, earley, xearley | |||||
class WithLexer: | class WithLexer: | ||||
def __init__(self, lexer_conf): | def __init__(self, lexer_conf): | ||||
@@ -21,6 +18,7 @@ class WithLexer: | |||||
else: | else: | ||||
return stream | return stream | ||||
class LALR(WithLexer): | class LALR(WithLexer): | ||||
def __init__(self, lexer_conf, parser_conf, options=None): | def __init__(self, lexer_conf, parser_conf, options=None): | ||||
WithLexer.__init__(self, lexer_conf) | WithLexer.__init__(self, lexer_conf) | ||||
@@ -32,6 +30,7 @@ class LALR(WithLexer): | |||||
tokens = self.lex(text) | tokens = self.lex(text) | ||||
return self.parser.parse(tokens) | return self.parser.parse(tokens) | ||||
class LALR_ContextualLexer: | class LALR_ContextualLexer: | ||||
def __init__(self, lexer_conf, parser_conf, options=None): | def __init__(self, lexer_conf, parser_conf, options=None): | ||||
self.lexer_conf = lexer_conf | self.lexer_conf = lexer_conf | ||||
@@ -49,6 +48,7 @@ class LALR_ContextualLexer: | |||||
tokens = self.lexer_conf.postlex.process(tokens) | tokens = self.lexer_conf.postlex.process(tokens) | ||||
return self.parser.parse(tokens, self.lexer.set_parser_state) | return self.parser.parse(tokens, self.lexer.set_parser_state) | ||||
def tokenize_text(text): | def tokenize_text(text): | ||||
new_text = [] | new_text = [] | ||||
line = 1 | line = 1 | ||||
@@ -224,6 +224,9 @@ class ApplyCallbacks(Transformer_NoRecurse): | |||||
return Tree(rule.origin, children) | return Tree(rule.origin, children) | ||||
def _compare_rules(rule1, rule2): | def _compare_rules(rule1, rule2): | ||||
if rule1 == rule2: | |||||
return 0 | |||||
if rule1.options and rule2.options: | if rule1.options and rule2.options: | ||||
if rule1.options.priority is not None and rule2.options.priority is not None: | if rule1.options.priority is not None and rule2.options.priority is not None: | ||||
assert rule1.options.priority != rule2.options.priority, "Priority is the same between both rules: %s == %s" % (rule1, rule2) | assert rule1.options.priority != rule2.options.priority, "Priority is the same between both rules: %s == %s" % (rule1, rule2) | ||||
@@ -15,7 +15,7 @@ class Tree(object): | |||||
def _pretty(self, level, indent_str): | def _pretty(self, level, indent_str): | ||||
if len(self.children) == 1 and not isinstance(self.children[0], Tree): | if len(self.children) == 1 and not isinstance(self.children[0], Tree): | ||||
return [ indent_str*level, self.data, '\t', '%s' % self.children[0], '\n'] | |||||
return [ indent_str*level, self._pretty_label(), '\t', '%s' % self.children[0], '\n'] | |||||
l = [ indent_str*level, self._pretty_label(), '\n' ] | l = [ indent_str*level, self._pretty_label(), '\n' ] | ||||
for n in self.children: | for n in self.children: | ||||