gitmirror/lark/indenter.py

"Provides Indentation services for languages with indentation similar to Python"

from abc import ABC, abstractmethod

from .exceptions import LarkError
from .lark import PostLex
from .lexer import Token

###{standalone
from typing import Tuple, List, Iterator, Optional

class DedentError(LarkError):
    pass

class Indenter(PostLex, ABC):
    def __init__(self) -> None:
        self.paren_level: Optional[int] = None
        self.indent_level: Optional[List[int]] = None
        assert self.tab_len > 0

    def handle_NL(self, token: Token) -> Iterator[Token]:
        if self.paren_level > 0:
            return

        yield token

        indent_str = token.rsplit('\n', 1)[1] # Tabs and spaces
        indent = indent_str.count(' ') + indent_str.count('\t') * self.tab_len

        if indent > self.indent_level[-1]:
            self.indent_level.append(indent)
            yield Token.new_borrow_pos(self.INDENT_type, indent_str, token)
        else:
            while indent < self.indent_level[-1]:
                self.indent_level.pop()
                yield Token.new_borrow_pos(self.DEDENT_type, indent_str, token)

            if indent != self.indent_level[-1]:
                raise DedentError('Unexpected dedent to column %s. Expected dedent to %s' % (indent, self.indent_level[-1]))

    def _process(self, stream):
        for token in stream:
            if token.type == self.NL_type:
                for t in self.handle_NL(token):
                    yield t
            else:
                yield token

            if token.type in self.OPEN_PAREN_types:
                self.paren_level += 1
            elif token.type in self.CLOSE_PAREN_types:
                self.paren_level -= 1
                assert self.paren_level >= 0

        while len(self.indent_level) > 1:
            self.indent_level.pop()
            yield Token(self.DEDENT_type, '')

        assert self.indent_level == [0], self.indent_level

    def process(self, stream):
        self.paren_level = 0
        self.indent_level = [0]
        return self._process(stream)

    # XXX Hack for ContextualLexer. Maybe there's a more elegant solution?
    @property
    def always_accept(self):
        return (self.NL_type,)

    @property
    @abstractmethod
    def NL_type(self) -> str:
        ...

    @property
    @abstractmethod
    def OPEN_PAREN_types(self) -> List[str]:
        ...

    @property
    @abstractmethod
    def CLOSE_PAREN_types(self) -> List[str]:
        ...

    @property
    @abstractmethod
    def INDENT_type(self) -> str:
        ...

    @property
    @abstractmethod
    def DEDENT_type(self) -> str:
        ...

    @property
    @abstractmethod
    def tab_len(self) -> int:
        ...

###}