|
- import re
- import sre_parse
-
- from .lexer import Lexer, ContextualLexer, Token
-
- from .common import is_terminal, GrammarError, ParserConf
- from .parsers import lalr_parser, earley, xearley, resolve_ambig
-
- class WithLexer:
- def init_traditional_lexer(self, lexer_conf):
- self.lexer_conf = lexer_conf
- self.lexer = Lexer(lexer_conf.tokens, ignore=lexer_conf.ignore)
-
- def init_contextual_lexer(self, lexer_conf, parser_conf):
- self.lexer_conf = lexer_conf
- d = {idx:t.keys() for idx, t in self.parser.analysis.parse_table.states.items()}
- always_accept = lexer_conf.postlex.always_accept if lexer_conf.postlex else ()
- self.lexer = ContextualLexer(lexer_conf.tokens, d, ignore=lexer_conf.ignore, always_accept=always_accept)
-
- def lex(self, text):
- stream = self.lexer.lex(text)
- if self.lexer_conf.postlex:
- return self.lexer_conf.postlex.process(stream)
- else:
- return stream
-
-
- class LALR(WithLexer):
- def __init__(self, lexer_conf, parser_conf, options=None):
- self.parser = lalr_parser.Parser(parser_conf)
- self.init_traditional_lexer(lexer_conf)
-
- def parse(self, text):
- token_stream = self.lex(text)
- return self.parser.parse(token_stream)
-
-
- class LALR_ContextualLexer(WithLexer):
- def __init__(self, lexer_conf, parser_conf, options=None):
- self.parser = lalr_parser.Parser(parser_conf)
- self.init_contextual_lexer(lexer_conf, parser_conf)
-
- def parse(self, text):
- token_stream = self.lex(text)
- return self.parser.parse(token_stream, self.lexer.set_parser_state)
-
- def get_ambiguity_resolver(options):
- if not options or options.ambiguity == 'resolve':
- return resolve_ambig.standard_resolve_ambig
- elif options.ambiguity == 'resolve__antiscore_sum':
- return resolve_ambig.antiscore_sum_resolve_ambig
- elif options.ambiguity == 'explicit':
- return None
- raise ValueError(options)
-
- def tokenize_text(text):
- line = 1
- col_start_pos = 0
- for i, ch in enumerate(text):
- if '\n' in ch:
- line += ch.count('\n')
- col_start_pos = i + ch.rindex('\n')
- yield Token('CHAR', ch, line=line, column=i - col_start_pos)
-
- class Earley_NoLex:
- def __init__(self, lexer_conf, parser_conf, options=None):
- self._prepare_match(lexer_conf)
-
- self.parser = earley.Parser(parser_conf, self.match,
- resolve_ambiguity=get_ambiguity_resolver(options))
-
-
- def match(self, term, text, index=0):
- return self.regexps[term].match(text, index)
-
- def _prepare_match(self, lexer_conf):
- self.regexps = {}
- for t in lexer_conf.tokens:
- regexp = t.pattern.to_regexp()
- width = sre_parse.parse(regexp).getwidth()
- if width != (1,1):
- raise GrammarError('Scanless parsing (lexer=None) requires all tokens to have a width of 1 (terminal %s: %s is %s)' % (sym, regexp, width))
- self.regexps[t.name] = re.compile(regexp)
-
- def parse(self, text):
- token_stream = tokenize_text(text)
- return self.parser.parse(token_stream)
-
- class Earley(WithLexer):
- def __init__(self, lexer_conf, parser_conf, options=None):
- self.init_traditional_lexer(lexer_conf)
-
- self.parser = earley.Parser(parser_conf, self.match,
- resolve_ambiguity=get_ambiguity_resolver(options))
-
- def match(self, term, token):
- return term == token.type
-
- def parse(self, text):
- tokens = self.lex(text)
- return self.parser.parse(tokens)
-
-
- class XEarley:
- def __init__(self, lexer_conf, parser_conf, options=None):
- self.token_by_name = {t.name:t for t in lexer_conf.tokens}
-
- self._prepare_match(lexer_conf)
-
- self.parser = xearley.Parser(parser_conf,
- self.match,
- resolve_ambiguity=get_ambiguity_resolver(options),
- ignore=lexer_conf.ignore,
- predict_all=options.earley__predict_all
- )
-
- def match(self, term, text, index=0):
- return self.regexps[term].match(text, index)
-
- def _prepare_match(self, lexer_conf):
- self.regexps = {}
- for t in lexer_conf.tokens:
- regexp = t.pattern.to_regexp()
- assert sre_parse.parse(regexp).getwidth()
- self.regexps[t.name] = re.compile(regexp)
-
- def parse(self, text):
- return self.parser.parse(text)
-
- def get_frontend(parser, lexer):
- if parser=='lalr':
- if lexer is None:
- raise ValueError('The LALR parser requires use of a lexer')
- elif lexer == 'standard':
- return LALR
- elif lexer == 'contextual':
- return LALR_ContextualLexer
- else:
- raise ValueError('Unknown lexer: %s' % lexer)
- elif parser=='earley':
- if lexer is None:
- return Earley_NoLex
- elif lexer=='standard':
- return Earley
- elif lexer=='dynamic':
- return XEarley
- elif lexer=='contextual':
- raise ValueError('The Earley parser does not support the contextual parser')
- else:
- raise ValueError('Unknown lexer: %s' % lexer)
- else:
- raise ValueError('Unknown parser: %s' % parser)
-
-
|