feat(apidocs): Build an internal OpenAPI spec of every endpoint method

.gitignore

@@ -41,6 +41,7 @@ node_modules
 /tests/apidocs/openapi-deprecated.json
 /tests/sentry/utils/appleconnect/*.json
 /tests/apidocs/openapi-spectacular.json
+/tests/apidocs/openapi-internal.json
 /fixtures/integrations/*/data/**
 /wheelhouse
 /test_cli/

Makefile

@@ -44,6 +44,12 @@ build-deprecated-docs:
 	@echo "--> Building deprecated openapi spec from json files"
 	pnpm run build-deprecated-docs
 
+# Every endpoint method, published or not, for internal consumers such as frontend type
+# generation; never published. See src/sentry/apidocs/internal/__init__.py.
+build-internal-api-docs: build-deprecated-docs
+	@echo "--> Building internal drf-spectacular openapi spec (every endpoint method)"
+	@OPENAPIGENERATE=1 sentry django spectacular --generator-class sentry.apidocs.internal.InternalGenerator --file tests/apidocs/openapi-internal.json --format openapi-json --validate
+
 build-api-docs: build-deprecated-docs build-spectacular-docs
 	@echo "--> Dereference the json schema for ease of use"
 	pnpm run deref-api-docs

src/sentry/apidocs/extensions.py

@@ -12,7 +12,7 @@
 from drf_spectacular.utils import Direction
 
 from sentry.api.serializers.rest_framework.base import snake_to_camel_case
-from sentry.apidocs.spectacular_ports import resolve_type_hint
+from sentry.apidocs.spectacular_ports import resolve_type_hint, resolve_type_hint_lenient
 from sentry.conf.server import GRANULAR_SCOPES
 
 
@@ -38,7 +38,9 @@ class TokenAuthExtension(_TypedExtension, OpenApiAuthenticationExtension):
     def get_security_requirement(self, auto_schema: AutoSchema) -> dict[str, list[Any]]:
         scopes = set()
         for permission in auto_schema.view.get_permissions():
-            for s in permission.scope_map.get(auto_schema.method, []):
+            # Permissions such as `SentryIsAuthenticated` require no scope and have no
+            # map. Only unpublished endpoints use them.
+            for s in getattr(permission, "scope_map", {}).get(auto_schema.method, []):
                 scopes.add(s)
 
         scope_list = sorted(scopes - GRANULAR_SCOPES)
@@ -88,7 +90,7 @@ def get_name(self, auto_schema: AutoSchema, direction: Direction) -> str | None:
         return name
 
     def map_serializer(self, auto_schema: AutoSchema, direction: Direction) -> Any:
-        return resolve_type_hint(self.target.typeSchema)
+        return resolve_type_hint_lenient(self.target.typeSchema, self.target.__name__)
 
 
 class SentryAppIssueActionSerializerExtension(_TypedExtension, OpenApiSerializerExtension):

src/sentry/apidocs/hooks.py

@@ -194,7 +194,21 @@ def _declared_silo(view_class: type) -> DeclaredSilo:
 EXPERIMENTAL_NOTICE = "**Experimental:** This API is under active development and may change."
 
 
-def custom_preprocessing_hook(endpoints: Any) -> Any:  # TODO: organize method, rename
+def _is_published(status: ApiPublishStatus) -> bool:
+    return status.is_published
+
+
+def custom_preprocessing_hook(  # TODO: organize method, rename
+    endpoints: Any,
+    *,
+    admit: Callable[[ApiPublishStatus], bool] = _is_published,
+    public_path: Callable[[str], str] | None = None,
+) -> Any:
+    """Check every endpoint, collect its metadata, and keep the methods `admit` lets in.
+
+    The exclusions and the ownership allowlist are written against the published build's
+    spelling of each path. A build that spells paths otherwise, such as the internal one
+    (`sentry.apidocs.internal`), passes `public_path` to give that spelling."""
     _ENDPOINT_SERVERS.clear()
     _EXPERIMENTAL_OPERATIONS.clear()
     _OPERATION_SILOS.clear()
@@ -203,6 +217,8 @@ def custom_preprocessing_hook(endpoints: Any) -> Any:  # TODO: organize method,
     filtered = []
     ownership_data: dict[ApiOwner, dict[ApiPublishStatus, set[str]]] = {}
     for path, path_regex, method, callback in endpoints:
+        spelling = path if public_path is None else public_path(path)
+
         # Collect servers from endpoint class for postprocessing
         endpoint_servers = getattr(callback.view_class, "servers", None)
         if endpoint_servers is not None:
@@ -219,7 +235,7 @@ def custom_preprocessing_hook(endpoints: Any) -> Any:  # TODO: organize method,
 
         # Fail if endpoint is unowned
         if owner_team == ApiOwner.UNOWNED:
-            if path not in API_OWNERSHIP_ALLOWLIST_DONT_MODIFY:
+            if spelling not in API_OWNERSHIP_ALLOWLIST_DONT_MODIFY:
                 raise SentryApiBuildError(
                     f"Endpoint {callback.view_class} is missing the attribute owner: ApiOwner. \n"
                     + "If you can't find your team in ApiOwners feel free to add the associated github group. ",
@@ -244,7 +260,7 @@ def custom_preprocessing_hook(endpoints: Any) -> Any:  # TODO: organize method,
                 f"Endpoint {callback.view_class} lists safe methods in idempotent_methods: {sorted(idempotent_methods & _SAFE_METHODS)}. Safe methods are always idempotent; only list non-safe methods such as POST.",
             )
 
-        if any(path.startswith(p) for p in EXCLUSION_PATH_PREFIXES):
+        if any(spelling.startswith(p) for p in EXCLUSION_PATH_PREFIXES):
             pass
 
         else:
@@ -254,12 +270,16 @@ def custom_preprocessing_hook(endpoints: Any) -> Any:  # TODO: organize method,
                 _IDEMPOTENT_OPERATIONS.add(operation_key)
             # endpoints that are documented via tooling
             status = callback.view_class.publish_status.get(method)
-            if status is not None and status.is_published:
-                if declared_idempotent is None and method not in _SAFE_METHODS:
+            if status is not None and admit(status):
+                if (
+                    status.is_published
+                    and declared_idempotent is None
+                    and method not in _SAFE_METHODS
+                ):
                     raise SentryApiBuildError(
                         f"Endpoint {callback.view_class} publishes {method} but does not declare idempotent_methods. List the methods a client may resend after a timeout or 5xx without changing the outcome, or set `idempotent_methods = frozenset()` if there are none. See src/sentry/apidocs/README.md.",
                     )
-                # only pass declared public methods of the endpoint
+                # only pass admitted (by default, published) methods of the endpoint
                 # to the rest of the OpenAPI build pipeline
                 filtered.append((path, path_regex, method, callback))
                 if status is ApiPublishStatus.PUBLIC_EXPERIMENTAL:
@@ -340,8 +360,18 @@ def _validate_request_body(
 
 
 def custom_postprocessing_hook(
-    result: dict[str, Any], generator: Any, **kwargs: Any
+    result: dict[str, Any],
+    generator: Any,
+    *,
+    in_reference: Callable[[Mapping[str, Any]], bool] | None = None,
+    **kwargs: Any,
 ) -> dict[str, Any]:
+    """Stamp operation metadata, then check what the published reference needs.
+
+    The published build describes only what the reference carries, so every operation
+    gets those checks. A build that describes more, such as the internal one
+    (`sentry.apidocs.internal`), passes `in_reference` to pick the operations that get
+    them; it describes its unpublished operations only for their shapes."""
     # Stamp before alias canonicalization so metadata stays with its operation.
     for path, operations in result["paths"].items():
         for method, operation in operations.items():
@@ -397,6 +427,9 @@ def custom_postprocessing_hook(
 
             endpoint_name = f"'{method_info['operationId']}'"
 
+            if in_reference is not None and not in_reference(method_info):
+                continue
+
             summary = method_info.get("summary")
             if summary is not None:
                 if summary in summaries_seen:

src/sentry/apidocs/internal/__init__.py

@@ -0,0 +1,49 @@
+"""The internal OpenAPI build: every endpoint method Sentry serves, published or not.
+
+The published build (`make build-spectacular-docs`) describes only published methods,
+for the public API reference. This one describes all of them, for internal consumers
+such as frontend type generation, and is never published. `make build-internal-api-docs`
+runs it, by handing drf-spectacular's `spectacular` command
+`--generator-class sentry.apidocs.internal.InternalGenerator`.
+
+It is the published build, with the same hooks, checks, and settings, except that:
+
+- Nothing is omitted. Fields kept out of the public reference with
+  `omit_from_public_schema` (and TypedDict `exclude_fields` that declare a reason) are
+  described, since the backend sends them. With no omissions, it builds once, without
+  the published build's second build checking them.
+- Every method is admitted, whatever its publish status, and stamped with it as
+  `x-sentry-publish-status`, and with the permission classes guarding it as
+  `x-sentry-permissions`. The published reference's own checks (tags, descriptions,
+  unique summaries) skip unpublished operations.
+- A literal alternation in a route, such as `(?:issues|groups)`, is spelled as its
+  first branch rather than `{var}`, so routes that differ only there stay apart. Every
+  path such a route answers on is listed under `x-sentry-path-aliases`.
+- A response no `@extend_schema` declares is read from the handler's `Response[T]`
+  annotation, or marked `x-sentry-undocumented`.
+- TypedDicts are closed objects, the null member of an optional union admits only null,
+  and a type hint that cannot be resolved becomes an `x-sentry-unresolved` placeholder
+  instead of failing the build.
+
+`InternalGenerator` switches all of this on for the duration of its own build, through
+parameters of the shared hooks and `spectacular_ports.TypeHintPolicy`, whose defaults are
+the published build's. The published build never sees any of it.
+"""
+
+from sentry.apidocs.internal.generator import (
+    PATH_ALIASES_EXTENSION,
+    PERMISSIONS_EXTENSION,
+    PUBLISH_STATUS_EXTENSION,
+    InternalGenerator,
+)
+from sentry.apidocs.internal.response_annotations import UNDOCUMENTED_EXTENSION
+from sentry.apidocs.spectacular_ports import UNRESOLVED_EXTENSION
+
+__all__ = (
+    "InternalGenerator",
+    "PATH_ALIASES_EXTENSION",
+    "PERMISSIONS_EXTENSION",
+    "PUBLISH_STATUS_EXTENSION",
+    "UNDOCUMENTED_EXTENSION",
+    "UNRESOLVED_EXTENSION",
+)