pygments: add typing annotation for missing base files of pygments

Author: lev-blit

stubs/Pygments/pygments/__init__.pyi

@@ -13,10 +13,10 @@ __all__ = ["lex", "format", "highlight"]
 
 def lex(code: str, lexer: Lexer) -> Iterator[tuple[_TokenType, str]]: ...
 @overload
-def format(tokens, formatter: Formatter[_T], outfile: SupportsWrite[_T]) -> None: ...
+def format(tokens: Iterator[tuple[_TokenType, str]], formatter: Formatter[_T], outfile: SupportsWrite[_T]) -> None: ...
 @overload
-def format(tokens, formatter: Formatter[_T], outfile: None = None) -> _T: ...
+def format(tokens: Iterator[tuple[_TokenType, str]], formatter: Formatter[_T], outfile: None = None) -> _T: ...
 @overload
-def highlight(code, lexer, formatter: Formatter[_T], outfile: SupportsWrite[_T]) -> None: ...
+def highlight(code: str, lexer: Lexer, formatter: Formatter[_T], outfile: SupportsWrite[_T]) -> None: ...
 @overload
-def highlight(code, lexer, formatter: Formatter[_T], outfile: None = None) -> _T: ...
+def highlight(code: str, lexer: Lexer, formatter: Formatter[_T], outfile: None = None) -> _T: ...

stubs/Pygments/pygments/cmdline.pyi

@@ -1,8 +1,10 @@
 import argparse
+import sys
+from collections.abc import Sequence
 
-def main_inner(parser, argns): ...
+def main_inner(parser: argparse.ArgumentParser, argns: argparse.Namespace) -> int: ...
 
 class HelpFormatter(argparse.HelpFormatter):
-    def __init__(self, prog, indent_increment: int = 2, max_help_position: int = 16, width=None) -> None: ...
+    def __init__(self, prog: str, indent_increment: int = 2, max_help_position: int = 16, width: int | None = None) -> None: ...
 
-def main(args=...): ...
+def main(args: Sequence[str] = sys.argv) -> int: ...

stubs/Pygments/pygments/console.pyi

@@ -1,10 +1,8 @@
-from typing import Any
-
 esc: str
-codes: Any
-dark_colors: Any
-light_colors: Any
+codes: dict[str, str]
+dark_colors: list[str]
+light_colors: list[str]
 
-def reset_color(): ...
-def colorize(color_key, text): ...
-def ansiformat(attr, text): ...
+def reset_color() -> str: ...
+def colorize(color_key: str, text: str) -> str: ...
+def ansiformat(attr: str, text: str) -> str: ...

stubs/Pygments/pygments/filter.pyi

@@ -1,18 +1,20 @@
-from collections.abc import Iterable, Iterator
+from collections.abc import Callable, Generator, Iterable, Iterator
 from typing import Any
 
 from pygments.lexer import Lexer
 from pygments.token import _TokenType
 
-def apply_filters(stream, filters, lexer=None): ...
-def simplefilter(f): ...
+def apply_filters(
+    stream: Callable[[], Iterator[tuple[_TokenType, str]]], filters: list[Filter], lexer: Lexer | None = None
+) -> Generator[tuple[_TokenType, str], None, tuple[_TokenType, str]]: ...
+def simplefilter(f: Callable[..., Any]) -> type[FunctionFilter]: ...
 
 class Filter:
     options: Any
-    def __init__(self, **options) -> None: ...
+    def __init__(self, **options: Any) -> None: ...
     def filter(self, lexer: Lexer, stream: Iterable[tuple[_TokenType, str]]) -> Iterator[tuple[_TokenType, str]]: ...
 
 class FunctionFilter(Filter):
-    function: Any
-    def __init__(self, **options) -> None: ...
+    function: Callable[..., Any] | None = None
+    def __init__(self, **options: Any) -> None: ...
     def filter(self, lexer: Lexer, stream: Iterable[tuple[_TokenType, str]]) -> Iterator[tuple[_TokenType, str]]: ...

stubs/Pygments/pygments/formatter.pyi

@@ -1,22 +1,27 @@
+from _io import _TextIOBase
+from collections.abc import Iterator
 from typing import Any, Generic, TypeVar, overload
 
+from pygments.style import Style
+from pygments.token import _TokenType
+
 _T = TypeVar("_T", str, bytes)
 
 class Formatter(Generic[_T]):
-    name: Any
-    aliases: Any
-    filenames: Any
+    name: str | None = None
+    aliases: list[str]
+    filenames: list[str]
     unicodeoutput: bool
-    style: Any
-    full: Any
-    title: Any
-    encoding: Any
-    options: Any
+    style: type[Style]
+    full: bool
+    title: str
+    encoding: str
+    options: dict[str, Any]
     @overload
-    def __init__(self: Formatter[str], *, encoding: None = None, outencoding: None = None, **options) -> None: ...
+    def __init__(self: Formatter[str], *, encoding: None = None, outencoding: None = None, **options: Any) -> None: ...
     @overload
-    def __init__(self: Formatter[bytes], *, encoding: str, outencoding: None = None, **options) -> None: ...
+    def __init__(self: Formatter[bytes], *, encoding: str, outencoding: None = None, **options: Any) -> None: ...
     @overload
-    def __init__(self: Formatter[bytes], *, encoding: None = None, outencoding: str, **options) -> None: ...
-    def get_style_defs(self, arg: str = ""): ...
-    def format(self, tokensource, outfile): ...
+    def __init__(self: Formatter[bytes], *, encoding: None = None, outencoding: str, **options: Any) -> None: ...
+    def get_style_defs(self, arg: str = "") -> str: ...
+    def format(self, tokensource: Iterator[tuple[_TokenType, str]], outfile: _TextIOBase) -> None: ...

stubs/Pygments/pygments/formatters/img.pyi

@@ -1,4 +1,4 @@
-from typing import Any, TypeVar
+from typing import Any, NoReturn, TypeVar
 
 from pygments.formatter import Formatter
 
@@ -46,7 +46,8 @@ class ImageFormatter(Formatter[_T]):
     hl_lines: Any
     hl_color: Any
     drawables: Any
-    def get_style_defs(self, arg: str = "") -> None: ...
+    # raises NotImplementedError
+    def get_style_defs(self, arg: str = "") -> NoReturn: ...
     def format(self, tokensource, outfile) -> None: ...
 
 class GifImageFormatter(ImageFormatter[_T]):

stubs/Pygments/pygments/lexer.pyi

@@ -1,13 +1,17 @@
+import re
 from _typeshed import Incomplete
-from collections.abc import Iterable, Iterator, Sequence
+from collections.abc import Callable, Iterable, Iterator, Sequence
 from re import RegexFlag
-from typing import ClassVar
+from typing import Any, ClassVar
 
+from pygments.filter import Filter
 from pygments.token import _TokenType
 from pygments.util import Future
 
+line_re: re.Pattern[str]
+
 class LexerMeta(type):
-    def __new__(cls, name, bases, d): ...
+    def __new__(cls, name: str, bases: tuple[type, ...], d: dict[str, Any]): ...
     def analyse_text(self, text: str) -> float: ...  # actually defined in class Lexer
     # ClassVars of Lexer, but same situation as with StyleMeta and Style
     name: str
@@ -19,83 +23,129 @@ class LexerMeta(type):
     url: str | None
 
 class Lexer(metaclass=LexerMeta):
-    options: Incomplete
-    stripnl: Incomplete
-    stripall: Incomplete
-    ensurenl: Incomplete
-    tabsize: Incomplete
-    encoding: Incomplete
-    filters: Incomplete
-    def __init__(self, **options) -> None: ...
-    def add_filter(self, filter_, **options) -> None: ...
+    options: dict[str, Any]
+    stripnl: bool
+    stripall: bool
+    ensurenl: bool
+    tabsize: int
+    encoding: str
+    filters: list[Filter]
+    def __init__(self, **options: Any) -> None: ...
+    def add_filter(self, filter_: Filter, **options: Any) -> None: ...
     def get_tokens(self, text: str, unfiltered: bool = False) -> Iterator[tuple[_TokenType, str]]: ...
     def get_tokens_unprocessed(self, text: str) -> Iterator[tuple[int, _TokenType, str]]: ...
 
 class DelegatingLexer(Lexer):
-    root_lexer: Incomplete
-    language_lexer: Incomplete
+    root_lexer: Lexer
+    language_lexer: Lexer
     needle: Incomplete
-    def __init__(self, _root_lexer, _language_lexer, _needle=..., **options) -> None: ...
+    def __init__(
+        self, _root_lexer: type[Lexer], _language_lexer: type[Lexer], _needle: _TokenType = ..., **options: Any
+    ) -> None: ...
     def get_tokens_unprocessed(self, text: str) -> Iterator[tuple[int, _TokenType, str]]: ...
 
 class include(str): ...
 class _inherit: ...
 
-inherit: Incomplete
+inherit: _inherit
 
-class combined(tuple[Incomplete, ...]):
-    def __new__(cls, *args): ...
-    def __init__(self, *args) -> None: ...
+class combined(tuple[str, ...]):
+    def __new__(cls, *args: str): ...
+    def __init__(self, *args: str) -> None: ...
 
 class _PseudoMatch:
-    def __init__(self, start, text) -> None: ...
-    def start(self, arg=None): ...
-    def end(self, arg=None): ...
-    def group(self, arg=None): ...
-    def groups(self): ...
-    def groupdict(self): ...
+    def __init__(self, start: int, text: str) -> None: ...
+    def start(self, arg=None) -> int: ...
+    def end(self, arg=None) -> int: ...
+    def group(self, arg=None) -> str: ...
+    def groups(self) -> tuple[str]: ...
+    def groupdict(self) -> dict[str, Any]: ...
 
-def bygroups(*args): ...
+def bygroups(
+    *args: _TokenType | Callable[[Lexer, _PseudoMatch, LexerContext], Iterator[tuple[int, _TokenType, str]]]
+) -> Callable[[Lexer, _PseudoMatch, LexerContext], Iterator[tuple[int, _TokenType, str]]]: ...
 
 class _This: ...
 
-this: Incomplete
+this: _This
 
-def using(_other, **kwargs): ...
+def using(
+    _other: _This | Lexer, **kwargs: Any
+) -> Callable[[Lexer, _PseudoMatch, LexerContext], Iterator[tuple[int, _TokenType, str]]]: ...
 
 class default:
-    state: Incomplete
-    def __init__(self, state) -> None: ...
+    state: str
+    def __init__(self, state: str) -> None: ...
 
 class words(Future):
-    words: Incomplete
-    prefix: Incomplete
-    suffix: Incomplete
-    def __init__(self, words, prefix: str = "", suffix: str = "") -> None: ...
-    def get(self): ...
+    words: Sequence[str]
+    prefix: str
+    suffix: str
+    def __init__(self, words: Sequence[str], prefix: str = "", suffix: str = "") -> None: ...
+    def get(self) -> str: ...
 
 class RegexLexerMeta(LexerMeta):
-    def process_tokendef(cls, name, tokendefs=None): ...
-    def get_tokendefs(cls): ...
-    def __call__(cls, *args, **kwds): ...
+    def process_tokendef(
+        cls,
+        name: str,
+        tokendefs: (
+            dict[
+                str,
+                list[
+                    tuple[str, _TokenType | Iterator[tuple[int, _TokenType, str]]]
+                    | tuple[str, _TokenType | Iterator[tuple[int, _TokenType, str]], str]
+                ],
+            ]
+            | None
+        ) = None,
+    ): ...
+    def get_tokendefs(
+        cls,
+    ) -> dict[
+        str,
+        list[
+            tuple[str, _TokenType | Iterator[tuple[int, _TokenType, str]]]
+            | tuple[str, _TokenType | Iterator[tuple[int, _TokenType, str]], str]
+        ],
+    ]: ...
+    def __call__(cls, *args: Any, **kwds: Any) -> Any: ...
 
 class RegexLexer(Lexer, metaclass=RegexLexerMeta):
     flags: ClassVar[RegexFlag]
-    tokens: ClassVar[dict[str, list[Incomplete]]]
+    tokens: ClassVar[
+        dict[
+            str,
+            list[
+                tuple[str, _TokenType | Iterator[tuple[int, _TokenType, str]]]
+                | tuple[str, _TokenType | Iterator[tuple[int, _TokenType, str]], str]
+            ],
+        ]
+    ]
     def get_tokens_unprocessed(self, text: str, stack: Iterable[str] = ("root",)) -> Iterator[tuple[int, _TokenType, str]]: ...
 
 class LexerContext:
-    text: Incomplete
-    pos: Incomplete
-    end: Incomplete
-    stack: Incomplete
-    def __init__(self, text, pos, stack=None, end=None) -> None: ...
+    text: str
+    pos: int
+    end: int
+    stack: list[str]
+    def __init__(self, text: str, pos: int, stack: list[str] | None = None, end: int | None = None) -> None: ...
 
 class ExtendedRegexLexer(RegexLexer):
     def get_tokens_unprocessed(  # type: ignore[override]
         self, text: str | None = None, context: LexerContext | None = None
     ) -> Iterator[tuple[int, _TokenType, str]]: ...
 
+def do_insertions(
+    insertions: list[tuple[int, list[tuple[int, _TokenType, str]]]],
+    tokens: dict[
+        str,
+        list[
+            tuple[str, _TokenType | Iterator[tuple[int, _TokenType, str]]]
+            | tuple[str, _TokenType | Iterator[tuple[int, _TokenType, str]], str]
+        ],
+    ],
+) -> Iterator[tuple[int, _TokenType, str]]: ...
+
 class ProfilingRegexLexerMeta(RegexLexerMeta): ...
 
 class ProfilingRegexLexer(RegexLexer, metaclass=ProfilingRegexLexerMeta):

stubs/Pygments/pygments/modeline.pyi

@@ -1 +1 @@
-def get_filetype_from_buffer(buf, max_lines: int = 5): ...
+def get_filetype_from_buffer(buf: str, max_lines: int = 5) -> str: ...

stubs/Pygments/pygments/regexopt.pyi

@@ -1,8 +1,10 @@
-from typing import Any
+import re
+from collections.abc import Sequence
+from operator import itemgetter
 
-CS_ESCAPE: Any
-FIRST_ELEMENT: Any
+CS_ESCAPE: re.Pattern[str]
+FIRST_ELEMENT: itemgetter[int]
 
-def make_charset(letters): ...
-def regex_opt_inner(strings, open_paren): ...
-def regex_opt(strings, prefix: str = "", suffix: str = ""): ...
+def make_charset(letters: Sequence[str]) -> str: ...
+def regex_opt_inner(strings: Sequence[str], open_paren: str) -> str: ...
+def regex_opt(strings: Sequence[str], prefix: str = "", suffix: str = "") -> str: ...

stubs/Pygments/pygments/scanner.pyi

@@ -1,19 +1,19 @@
-from typing import Any
+import re
 
 class EndOfText(RuntimeError): ...
 
 class Scanner:
-    data: Any
-    data_length: Any
+    data: str
+    data_length: int
     start_pos: int
     pos: int
-    flags: Any
-    last: Any
-    match: Any
-    def __init__(self, text, flags: int = 0) -> None: ...
+    flags: int
+    last: str
+    match: str
+    def __init__(self, text: str, flags: int = 0) -> None: ...
     @property
-    def eos(self): ...
-    def check(self, pattern): ...
-    def test(self, pattern): ...
-    def scan(self, pattern): ...
+    def eos(self) -> bool: ...
+    def check(self, pattern: str) -> re.Match[str] | None: ...
+    def test(self, pattern: str) -> bool: ...
+    def scan(self, pattern: str) -> bool: ...
     def get_char(self) -> None: ...