Uncouple doctype transformation

src/docstub-stubs/_analysis.pyi

@@ -14,6 +14,7 @@ from typing import Any, ClassVar
 import libcst as cst
 import libcst.matchers as cstm
 
+from ._report import Stats
 from ._utils import accumulate_qualname, module_name_from_path, pyfile_checksum
 
 logger: logging.Logger
@@ -83,6 +84,7 @@ class TypeMatcher:
         types: dict[str, PyImport] | None = ...,
         type_prefixes: dict[str, PyImport] | None = ...,
         type_nicknames: dict[str, str] | None = ...,
+        stats: Stats | None = ...,
     ) -> None: ...
     def _resolve_nickname(self, name: str) -> str: ...
     def match(self, search: str) -> tuple[str | None, PyImport | None]: ...

src/docstub-stubs/_app_generate_stubs.pyi

@@ -18,9 +18,8 @@ from ._path_utils import (
     walk_source_and_targets,
     walk_source_package,
 )
-from ._report import setup_logging
+from ._report import Stats, setup_logging
 from ._stubs import Py2StubTransformer, try_format_stub
-from ._utils import update_with_add_values
 from ._version import __version__
 
 logger: logging.Logger

src/docstub-stubs/_docstrings.pyi

@@ -2,32 +2,26 @@
 
 import logging
 import traceback
+import warnings
 from collections.abc import Generator, Iterable
 from dataclasses import dataclass, field
 from functools import cached_property
-from pathlib import Path
-from typing import Any, ClassVar
 
 import click
 import lark
 import lark.visitors
 import numpydoc.docscrape as npds
 
 from ._analysis import PyImport, TypeMatcher
-from ._report import ContextReporter
-from ._utils import DocstubError, escape_qualname
+from ._doctype import BlacklistedQualname, Expr, Term, TermKind, parse_doctype
+from ._report import ContextReporter, Stats
+from ._utils import escape_qualname
 
 logger: logging.Logger
 
-here: Path
-grammar_path: Path
-
-with grammar_path.open() as file:
-    _grammar: str
-
-_lark: lark.Lark
-
-def _find_one_token(tree: lark.Tree, *, name: str) -> lark.Token: ...
+def _update_qualnames(
+    expr: Expr, *, _parents: tuple[Expr, ...] = ...
+) -> Generator[tuple[tuple[Expr, ...], Term], str]: ...
 @dataclass(frozen=True, slots=True, kw_only=True)
 class Annotation:
 
@@ -54,57 +48,31 @@ class Annotation:
 
 FallbackAnnotation: Annotation
 
-class QualnameIsKeyword(DocstubError):
-    pass
-
-class DoctypeTransformer(lark.visitors.Transformer):
-    matcher: TypeMatcher
-    stats: dict[str, Any]
-
-    blacklisted_qualnames: ClassVar[frozenset[str]]
-
-    def __init__(
-        self, *, matcher: TypeMatcher | None = ..., **kwargs: dict[Any, Any]
-    ) -> None: ...
-    def doctype_to_annotation(
-        self, doctype: str, *, reporter: ContextReporter | None = ...
-    ) -> tuple[Annotation, list[tuple[str, int, int]]]: ...
-    def qualname(self, tree: lark.Tree) -> lark.Token: ...
-    def rst_role(self, tree: lark.Tree) -> lark.Token: ...
-    def union(self, tree: lark.Tree) -> str: ...
-    def subscription(self, tree: lark.Tree) -> str: ...
-    def param_spec(self, tree: lark.Tree) -> str: ...
-    def callable(self, tree: lark.Tree) -> str: ...
-    def natlang_literal(self, tree: lark.Tree) -> str: ...
-    def natlang_container(self, tree: lark.Tree) -> str: ...
-    def natlang_array(self, tree: lark.Tree) -> str: ...
-    def array_name(self, tree: lark.Tree) -> lark.Token: ...
-    def shape(self, tree: lark.Tree) -> lark.visitors._DiscardType: ...
-    def optional_info(self, tree: lark.Tree) -> lark.visitors._DiscardType: ...
-    def __default__(
-        self, data: lark.Token, children: list[lark.Token], meta: lark.tree.Meta
-    ) -> lark.Token | list[lark.Token]: ...
-    def _match_import(self, qualname: str, *, meta: lark.tree.Meta) -> str: ...
-
 def _uncombine_numpydoc_params(
     params: list[npds.Parameter],
 ) -> Generator[npds.Parameter]: ...
+def _red_partial_underline(doctype: str, *, start: int, stop: int) -> str: ...
+def doctype_to_annotation(
+    doctype: str,
+    *,
+    matcher: TypeMatcher | None = ...,
+    reporter: ContextReporter | None = ...,
+    stats: Stats | None = ...,
+) -> Annotation: ...
 
 class DocstringAnnotations:
     docstring: str
-    transformer: DoctypeTransformer
+    matcher: TypeMatcher
     reporter: ContextReporter
 
     def __init__(
         self,
         docstring: str,
         *,
-        transformer: DoctypeTransformer,
+        matcher: TypeMatcher | None = ...,
         reporter: ContextReporter | None = ...,
+        stats: Stats | None = ...,
     ) -> None: ...
-    def _doctype_to_annotation(
-        self, doctype: str, ds_line: int = ...
-    ) -> Annotation: ...
     @cached_property
     def attributes(self) -> dict[str, Annotation]: ...
     @cached_property

src/docstub-stubs/_doctype.pyi

@@ -0,0 +1,95 @@
+# File generated with docstub
+
+import enum
+import keyword
+import logging
+from collections.abc import Generator, Iterable, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from textwrap import indent
+from typing import Any, Final, Self
+
+import lark
+import lark.visitors
+from _typeshed import Incomplete
+
+from ._report import ContextReporter
+from ._utils import DocstubError
+
+logger: Final[logging.Logger]
+
+grammar_path: Final[Path]
+
+with grammar_path.open() as file:
+    _grammar: Final[str]
+
+_lark: Final[lark.Lark]
+
+def flatten_recursive(iterable: Iterable[Iterable | str]) -> Generator[str]: ...
+def insert_between(iterable: Iterable, *, sep: Any) -> list[Any]: ...
+
+class TermKind(enum.StrEnum):
+
+    NAME = enum.auto()
+    LITERAL = enum.auto()
+    SYNTAX = enum.auto()
+
+class Term(str):
+    kind: TermKind
+    pos: tuple[int, int] | None
+
+    __slots__: Final[tuple[str, ...]]
+
+    def __new__(
+        cls, value: str, *, kind: TermKind | str, pos: tuple[int, int] | None = ...
+    ) -> Self: ...
+    def __repr__(self) -> str: ...
+    def __getnewargs_ex__(self) -> tuple[tuple[Any, ...], dict[str, Any]]: ...
+
+@dataclass(slots=True)
+class Expr:
+
+    rule: str
+    children: list[Self | Term]
+
+    @property
+    def terms(self) -> list[Term]: ...
+    @property
+    def names(self) -> list[Term]: ...
+    @property
+    def sub_expressions(self) -> list[Self]: ...
+    def __iter__(self) -> Generator[Expr | Term]: ...
+    def format_tree(self) -> str: ...
+    def print_tree(self) -> None: ...
+    def __repr__(self) -> str: ...
+    def __str__(self) -> str: ...
+    def as_code(self) -> str: ...
+
+BLACKLISTED_QUALNAMES: Final[set[str]]
+
+class BlacklistedQualname(DocstubError):
+    pass
+
+class DoctypeTransformer(lark.visitors.Transformer):
+    def __init__(self, *, reporter: ContextReporter | None = ...) -> None: ...
+    def start(self, tree: lark.Tree) -> Expr: ...
+    def qualname(self, tree: lark.Tree) -> Term: ...
+    def rst_role(self, tree: lark.Tree) -> Expr: ...
+    def ELLIPSES(self, token: lark.Token) -> Term: ...
+    def union(self, tree: lark.Tree) -> Expr: ...
+    def subscription(self, tree: lark.Tree) -> Expr: ...
+    def param_spec(self, tree: lark.Tree) -> Expr: ...
+    def callable(self, tree: lark.Tree) -> Expr: ...
+    def literal(self, tree: lark.Tree) -> Expr: ...
+    def natlang_literal(self, tree: lark.Tree) -> Expr: ...
+    def literal_item(self, tree: lark.Tree) -> Term: ...
+    def natlang_container(self, tree: lark.Tree) -> Expr: ...
+    def natlang_array(self, tree: lark.Tree) -> Expr: ...
+    def array_name(self, tree: lark.Tree) -> Term: ...
+    def dtype(self, tree: lark.Tree) -> Expr: ...
+    def shape(self, tree: lark.Tree) -> lark.visitors._DiscardType: ...
+    def optional_info(self, tree: lark.Tree) -> lark.visitors._DiscardType: ...
+    def extra_info(self, tree: lark.Tree) -> lark.visitors._DiscardType: ...
+    def _format_subscription(self, sequence: Sequence[str], *, rule: str) -> Expr: ...
+
+def parse_doctype(doctype: str, *, reporter: ContextReporter | None = ...) -> Expr: ...

src/docstub-stubs/_report.pyi

@@ -2,6 +2,7 @@
 
 import dataclasses
 import logging
+from collections.abc import Hashable, Iterator, Mapping, Sequence
 from pathlib import Path
 from textwrap import indent
 from typing import Any, ClassVar, Literal, Self, TextIO
@@ -33,23 +34,43 @@ class ContextReporter:
         short: str,
         *args: Any,
         log_level: int,
-        details: str | None = ...,
+        details: str | tuple[Any, ...] | None = ...,
         **log_kw: Any
     ) -> None: ...
     def debug(
-        self, short: str, *args: Any, details: str | None = ..., **log_kw: Any
+        self,
+        short: str,
+        *args: Any,
+        details: str | tuple[Any, ...] | None = ...,
+        **log_kw: Any
     ) -> None: ...
     def info(
-        self, short: str, *args: Any, details: str | None = ..., **log_kw: Any
+        self,
+        short: str,
+        *args: Any,
+        details: str | tuple[Any, ...] | None = ...,
+        **log_kw: Any
     ) -> None: ...
     def warn(
-        self, short: str, *args: Any, details: str | None = ..., **log_kw: Any
+        self,
+        short: str,
+        *args: Any,
+        details: str | tuple[Any, ...] | None = ...,
+        **log_kw: Any
     ) -> None: ...
     def error(
-        self, short: str, *args: Any, details: str | None = ..., **log_kw: Any
+        self,
+        short: str,
+        *args: Any,
+        details: str | tuple[Any, ...] | None = ...,
+        **log_kw: Any
     ) -> None: ...
     def critical(
-        self, short: str, *args: Any, details: str | None = ..., **log_kw: Any
+        self,
+        short: str,
+        *args: Any,
+        details: str | tuple[Any, ...] | None = ...,
+        **log_kw: Any
     ) -> None: ...
     def __post_init__(self) -> None: ...
     @staticmethod
@@ -80,3 +101,22 @@ class LogCounter(logging.NullHandler):
 def setup_logging(
     *, verbosity: Literal[-2, -1, 0, 1, 2, 3], group_errors: bool
 ) -> tuple[ReportHandler, LogCounter]: ...
+def update_with_add_values(
+    *mappings: Mapping[Hashable, int | Sequence], out: dict | None = ...
+) -> dict: ...
+
+class Stats(Mapping):
+    class _UNSET:
+        pass
+
+    def __init__(self, stats: dict[str, list[Any] | str] | None = ...) -> None: ...
+    def __getitem__(self, key: str) -> list[Any] | int: ...
+    def __iter__(self) -> Iterator: ...
+    def __len__(self) -> int: ...
+    def inc_counter(self, key: str, *, inc: int = ...) -> None: ...
+    def append_to_list(self, key: str, value: Any) -> None: ...
+    @classmethod
+    def merge(cls, *stats: Self) -> Self: ...
+    def __repr__(self) -> str: ...
+    def pop(self, key: str, *, default: Any = ...) -> list[Any] | int: ...
+    def pop_all(self) -> dict[str, list[Any] | int]: ...

src/docstub-stubs/_stubs.pyi

@@ -13,14 +13,10 @@ import libcst.matchers as cstm
 from _typeshed import Incomplete
 
 from ._analysis import PyImport, TypeMatcher
-from ._docstrings import (
-    Annotation,
-    DocstringAnnotations,
-    DoctypeTransformer,
-    FallbackAnnotation,
-)
-from ._report import ContextReporter
-from ._utils import module_name_from_path, update_with_add_values
+from ._docstrings import Annotation, DocstringAnnotations, FallbackAnnotation
+from ._doctype import DoctypeTransformer
+from ._report import ContextReporter, Stats
+from ._utils import module_name_from_path
 
 logger: logging.Logger
 
@@ -77,9 +73,6 @@ class Py2StubTransformer(cst.CSTTransformer):
     @property
     def is_inside_function_def(self) -> bool: ...
     def python_to_stub(self, source: str, *, module_path: Path | None = ...) -> str: ...
-    def collect_stats(
-        self, *, reset_after: bool = ...
-    ) -> dict[str, int | list[str]]: ...
     def visit_ClassDef(self, node: cst.ClassDef) -> Literal[True]: ...
     def leave_ClassDef(
         self, original_node: cst.ClassDef, updated_node: cst.ClassDef

src/docstub-stubs/_utils.pyi

@@ -2,7 +2,7 @@
 
 import itertools
 import re
-from collections.abc import Callable, Hashable, Mapping, Sequence
+from collections.abc import Callable
 from functools import lru_cache, wraps
 from pathlib import Path
 from typing import Any
@@ -15,9 +15,6 @@ def _resolve_path_before_caching(
 ) -> Callable[[Path], str]: ...
 def module_name_from_path(path: Path) -> str: ...
 def pyfile_checksum(path: Path) -> str: ...
-def update_with_add_values(
-    *mappings: Mapping[Hashable, int | Sequence], out: dict | None = ...
-) -> dict: ...
 
 class DocstubError(Exception):
     pass